[dpdk-dev] [PATCH 00/12] Add PCIe AER disable and IRQ support for ipn3ke

[dpdk-dev] [PATCH 00/12] Add PCIe AER disable and IRQ support for ipn3ke

[Rosen Xu]

This patch set adds PCIe AER disable and IRQ support for ipn3ke.
Disable PCIe AER is very useful when FPGA reload. IRQ is used
very widely in interrupt process.

For ipn3ke is connect to CPU with PCIe switch, driver needs to
scan all PCIe devices of ipn3ke, it also can get all i40e of card,
so ipn3ke driver doesn't need to take some configuration of i40e.

Rosen Xu (3):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga_rawdev: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei Zhang (2):
  raw/ifpga_rawdev/base: align the send buffer for SPI
  raw/ifpga_rawdev/base: introducing sensor APIs

Tianfei zhang (7):
  raw/ifpga_rawdev/base: add irq support
  raw/ifpga_rawdev/base: clear pending bit
  raw/ifpga_rawdev/base: add SEU error support
  raw/ifpga_rawdev/base: add device tree support
  raw/ifpga_rawdev/base: add sensor support
  raw/ifpga_rawdev/base: update SEU register definition
  raw/ifpga_rawdev: add SEU error handler

 drivers/net/i40e/base/i40e_type.h                  |   3 +
 drivers/net/i40e/i40e_ethdev.c                     |  34 +-
 drivers/net/i40e/rte_pmd_i40e.h                    |   4 +
 drivers/net/ipn3ke/Makefile                        |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c                 | 289 +-------
 drivers/net/ipn3ke/ipn3ke_representor.c            |   7 +-
 drivers/raw/ifpga_rawdev/base/ifpga_api.c          |  10 +
 drivers/raw/ifpga_rawdev/base/ifpga_defines.h      |  18 +-
 drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c  |  61 ++
 drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h  |   3 +
 drivers/raw/ifpga_rawdev/base/ifpga_fme.c          |  21 +
 drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c    |  69 +-
 drivers/raw/ifpga_rawdev/base/ifpga_port.c         |  20 +
 drivers/raw/ifpga_rawdev/base/ifpga_port_error.c   |  21 +
 drivers/raw/ifpga_rawdev/base/opae_hw_api.c        | 115 ++++
 drivers/raw/ifpga_rawdev/base/opae_hw_api.h        |  16 +
 drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h  |   2 +
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.c   | 462 +++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.h   |  66 ++
 drivers/raw/ifpga_rawdev/base/opae_osdep.h         |   7 +-
 .../raw/ifpga_rawdev/base/opae_spi_transaction.c   |  40 +-
 drivers/raw/ifpga_rawdev/ifpga_rawdev.c            | 764 ++++++++++++++++++++-
 drivers/raw/ifpga_rawdev/ifpga_rawdev.h            |  16 +
 mk/rte.app.mk                                      |   2 +-
 24 files changed, 1775 insertions(+), 277 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH 01/12] net/i40e: i40e support ipn3ke FPGA port bonding

[Rosen Xu]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/net/i40e/base/i40e_type.h |  3 +++
 drivers/net/i40e/i40e_ethdev.c    | 34 ++++++++++++++++++++++++++++++++--
 drivers/net/i40e/rte_pmd_i40e.h   |  4 ++++
 3 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..a4d46d8 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	//switch device
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 4e40b7a..e981256 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1312,6 +1312,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	//Update switch device pointer
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2782,6 +2785,20 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	}
 }
 
+void
+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
+struct rte_eth_dev *switch_dev)
+{
+	struct i40e_hw *hw;
+
+	if (!i40e_dev)
+		return;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+
+	hw->switch_dev = switch_dev;
+}
+
 int
 i40e_dev_link_update(struct rte_eth_dev *dev,
 		     int wait_to_complete)
@@ -2790,6 +2807,7 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	struct rte_eth_link link;
 	bool enable_lse = dev->data->dev_conf.intr_conf.lsc ? true : false;
 	int ret;
+	struct rte_eth_dev *switch_ethdev;
 
 	memset(&link, 0, sizeof(link));
 
@@ -2803,6 +2821,18 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	switch_ethdev = hw->switch_dev;
+	if (switch_ethdev) {
+		rte_eth_linkstatus_get(switch_ethdev, &link);
+		printf(">>>>>>>>>>>>>i40e_update_link 5 link.link_status %d\n",
+			link.link_status);
+	} else {
+		link.link_duplex = ETH_LINK_FULL_DUPLEX;
+		link.link_autoneg = ETH_LINK_SPEED_FIXED;
+		link.link_speed = ETH_SPEED_NUM_25G;
+		link.link_status = 0;
+	}
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
@@ -12541,7 +12571,7 @@ struct i40e_customized_pctype*
  *	b.	Old_filter = 10 (Stag_Inner_Vlan)
  *	c.	New_filter = 0x10
  *	d.	TR bit = 0xff (optional, not used here)
- *	e.	Buffer – 2 entries:
+ *	e.	Buffer - 2 entries:
  *		i.	Byte 0 = 8 (outer vlan FV index).
  *			Byte 1 = 0 (rsv)
  *			Byte 2-3 = 0x0fff
@@ -12555,7 +12585,7 @@ struct i40e_customized_pctype*
  *	a.	Valid_flags.replace_cloud = 1
  *	b.	Old_filter = 1 (instead of outer IP)
  *	c.	New_filter = 0x10
- *	d.	Buffer – 2 entries:
+ *	d.	Buffer - 2 entries:
  *		i.	Byte 0 = 0x80 | 7 (valid | Stag).
  *			Byte 1-3 = 0 (rsv)
  *		ii.	Byte 8 = 0x80 | 0x10 (valid | new l1 filter step1)
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..9d77c85 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,8 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+void
+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
+struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 00/12] Add PCIe AER disable and IRQ support for ipn3ke

[Rosen Xu]

This patch set adds PCIe AER disable and IRQ support for ipn3ke.
Disable PCIe AER is very useful when FPGA reload. IRQ is used
very widely in interrupt process.

For ipn3ke is connect to CPU with PCIe switch, driver needs to
scan all PCIe devices of ipn3ke, it also can get all i40e of card,
so ipn3ke driver doesn't need to take some configuration of i40e.

v2 updates:
===========
 - Add AUX feature support

Rosen Xu (3):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga_rawdev: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei Zhang (2):
  raw/ifpga_rawdev/base: align the send buffer for SPI
  raw/ifpga_rawdev/base: introducing sensor APIs

Tianfei zhang (7):
  raw/ifpga_rawdev/base: add irq support
  raw/ifpga_rawdev/base: clear pending bit
  raw/ifpga_rawdev/base: add SEU error support
  raw/ifpga_rawdev/base: add device tree support
  raw/ifpga_rawdev/base: add sensor support
  raw/ifpga_rawdev/base: update SEU register definition
  raw/ifpga_rawdev: add SEU error handler

 drivers/net/i40e/base/i40e_type.h                  |   3 +
 drivers/net/i40e/i40e_ethdev.c                     |  34 +-
 drivers/net/i40e/rte_pmd_i40e.h                    |   4 +
 drivers/net/ipn3ke/Makefile                        |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c                 | 289 +-------
 drivers/net/ipn3ke/ipn3ke_representor.c            |   7 +-
 drivers/raw/ifpga_rawdev/base/ifpga_api.c          |  10 +
 drivers/raw/ifpga_rawdev/base/ifpga_defines.h      |  18 +-
 drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c  |  61 ++
 drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h  |   3 +
 drivers/raw/ifpga_rawdev/base/ifpga_fme.c          |  21 +
 drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c    |  69 +-
 drivers/raw/ifpga_rawdev/base/ifpga_port.c         |  20 +
 drivers/raw/ifpga_rawdev/base/ifpga_port_error.c   |  21 +
 drivers/raw/ifpga_rawdev/base/opae_hw_api.c        | 115 +++
 drivers/raw/ifpga_rawdev/base/opae_hw_api.h        |  16 +
 drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h  |   2 +
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.c   | 462 ++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.h   |  66 ++
 drivers/raw/ifpga_rawdev/base/opae_osdep.h         |   7 +-
 .../raw/ifpga_rawdev/base/opae_spi_transaction.c   |  40 +-
 drivers/raw/ifpga_rawdev/ifpga_rawdev.c            | 795 ++++++++++++++++++++-
 drivers/raw/ifpga_rawdev/ifpga_rawdev.h            |  16 +
 mk/rte.app.mk                                      |   2 +-
 24 files changed, 1805 insertions(+), 278 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v2 01/12] net/i40e: i40e support ipn3ke FPGA port bonding

[Rosen Xu]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/net/i40e/base/i40e_type.h |  3 +++
 drivers/net/i40e/i40e_ethdev.c    | 34 ++++++++++++++++++++++++++++++++--
 drivers/net/i40e/rte_pmd_i40e.h   |  4 ++++
 3 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..a4d46d8 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	//switch device
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 4e40b7a..e981256 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1312,6 +1312,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	//Update switch device pointer
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2782,6 +2785,20 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	}
 }
 
+void
+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
+struct rte_eth_dev *switch_dev)
+{
+	struct i40e_hw *hw;
+
+	if (!i40e_dev)
+		return;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+
+	hw->switch_dev = switch_dev;
+}
+
 int
 i40e_dev_link_update(struct rte_eth_dev *dev,
 		     int wait_to_complete)
@@ -2790,6 +2807,7 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	struct rte_eth_link link;
 	bool enable_lse = dev->data->dev_conf.intr_conf.lsc ? true : false;
 	int ret;
+	struct rte_eth_dev *switch_ethdev;
 
 	memset(&link, 0, sizeof(link));
 
@@ -2803,6 +2821,18 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	switch_ethdev = hw->switch_dev;
+	if (switch_ethdev) {
+		rte_eth_linkstatus_get(switch_ethdev, &link);
+		printf(">>>>>>>>>>>>>i40e_update_link 5 link.link_status %d\n",
+			link.link_status);
+	} else {
+		link.link_duplex = ETH_LINK_FULL_DUPLEX;
+		link.link_autoneg = ETH_LINK_SPEED_FIXED;
+		link.link_speed = ETH_SPEED_NUM_25G;
+		link.link_status = 0;
+	}
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
@@ -12541,7 +12571,7 @@ struct i40e_customized_pctype*
  *	b.	Old_filter = 10 (Stag_Inner_Vlan)
  *	c.	New_filter = 0x10
  *	d.	TR bit = 0xff (optional, not used here)
- *	e.	Buffer – 2 entries:
+ *	e.	Buffer - 2 entries:
  *		i.	Byte 0 = 8 (outer vlan FV index).
  *			Byte 1 = 0 (rsv)
  *			Byte 2-3 = 0x0fff
@@ -12555,7 +12585,7 @@ struct i40e_customized_pctype*
  *	a.	Valid_flags.replace_cloud = 1
  *	b.	Old_filter = 1 (instead of outer IP)
  *	c.	New_filter = 0x10
- *	d.	Buffer – 2 entries:
+ *	d.	Buffer - 2 entries:
  *		i.	Byte 0 = 0x80 | 7 (valid | Stag).
  *			Byte 1-3 = 0 (rsv)
  *		ii.	Byte 8 = 0x80 | 0x10 (valid | new l1 filter step1)
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..9d77c85 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,8 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+void
+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
+struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 02/12] raw/ifpga_rawdev/base: add irq support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME globle error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c | 61 +++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c   | 22 ++++++++
 drivers/raw/ifpga_rawdev/base/ifpga_port.c        | 20 ++++++++
 drivers/raw/ifpga_rawdev/base/ifpga_port_error.c  | 21 ++++++++
 4 files changed, 124 insertions(+)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
index 63c8bcc..6b942e6 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,63 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+/*
+ * FIXME: we should get msix vec count during pci enumeration instead of
+ * below hardcode value.
+ */
+#define FPGA_MSIX_VEC_COUNT	20
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	irq_set->count = count;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
+
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
index 3794564..068f52c 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
@@ -373,9 +373,31 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set =
+			(struct fpga_fme_err_irq_set *)irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	spinlock_lock(&fme->lock);
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ)) {
+		spinlock_unlock(&fme->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_port.c b/drivers/raw/ifpga_rawdev/base/ifpga_port.c
index 6c41164..56b04a6 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_port.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_port.c
@@ -384,9 +384,29 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	spinlock_lock(&port->lock);
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ)) {
+		spinlock_unlock(&port->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c b/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c
index 138284e..8aef7d7 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c
@@ -136,9 +136,30 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	spinlock_lock(&port->lock);
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ)) {
+		spinlock_unlock(&port->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v2 02/12] raw/ifpga_rawdev/base: add irq support

[Jerin Jacob Kollanukkaran]

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Rosen Xu
> Sent: Friday, August 2, 2019 6:49 AM
> To: dev@dpdk.org
> Cc: ferruh.yigit@intel.com; tianfei.zhang@intel.com; rosen.xu@intel.com;
> andy.pei@intel.com; david.lomartire@intel.com; qi.z.zhang@intel.com;
> xiaolong.ye@intel.com
> Subject: [dpdk-dev] [PATCH v2 02/12] raw/ifpga_rawdev/base: add irq
> support
> 
> From: Tianfei zhang <tianfei.zhang@intel.com>
> 
> Add irq support for ifpga FME globle error, port error and uint unit.
> We implmented this feature by vfio interrupt mechanism.
> 
> Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
> ---
>  drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c | 61
> +++++++++++++++++++++++
>  drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c   | 22 ++++++++
>  drivers/raw/ifpga_rawdev/base/ifpga_port.c        | 20 ++++++++
>  drivers/raw/ifpga_rawdev/base/ifpga_port_error.c  | 21 ++++++++
>  4 files changed, 124 insertions(+)
> 
> diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
> b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
> index 63c8bcc..6b942e6 100644
> --- a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
> +++ b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
> @@ -3,6 +3,7 @@
>   */
> 
>  #include <sys/ioctl.h>
> +#include <rte_vfio.h>
> 
>  #include "ifpga_feature_dev.h"
> 
> @@ -331,3 +332,63 @@ int port_hw_init(struct ifpga_port_hw *port)
>  	port_hw_uinit(port);
>  	return ret;
>  }
> +
> +/*
> + * FIXME: we should get msix vec count during pci enumeration instead
> +of
> + * below hardcode value.
> + */
> +#define FPGA_MSIX_VEC_COUNT	20
> +/* irq set buffer length for interrupt */ #define MSIX_IRQ_SET_BUF_LEN
> +(sizeof(struct vfio_irq_set) + \
> +				sizeof(int) * FPGA_MSIX_VEC_COUNT)
> +
> +/* only support msix for now*/
> +static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
> +				  unsigned int count, s32 *fds)

Isn't better to use generic EAL function for the same?

> +{
> +	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
> +	struct vfio_irq_set *irq_set;
> +	int len, ret;
> +	int *fd_ptr;
> +
> +	len = sizeof(irq_set_buf);
> +
> +	irq_set = (struct vfio_irq_set *)irq_set_buf;
> +	irq_set->argsz = len;
> +	irq_set->count = count;
> +	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
> +				VFIO_IRQ_SET_ACTION_TRIGGER;
> +	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
> +	irq_set->start = vec_start;
> +
> +	fd_ptr = (int *)&irq_set->data;
> +	memcpy(fd_ptr, fds, sizeof(int) * count);
> +
> +	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
> +	if (ret)
> +		printf("Error enabling MSI-X interrupts\n");
> +
> +	return ret;
> +}
> +

Re: [dpdk-dev] [PATCH v2 02/12] raw/ifpga_rawdev/base: add irq support

[Zhang, Tianfei]

> -----Original Message-----
> From: Jerin Jacob Kollanukkaran [mailto:jerinj@marvell.com]
> Sent: Friday, August 2, 2019 11:58 AM
> To: Xu, Rosen <rosen.xu@intel.com>; dev@dpdk.org
> Cc: Yigit, Ferruh <ferruh.yigit@intel.com>; Zhang, Tianfei
> <tianfei.zhang@intel.com>; Pei, Andy <andy.pei@intel.com>; Lomartire,
> David <david.lomartire@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Ye,
> Xiaolong <xiaolong.ye@intel.com>
> Subject: RE: [dpdk-dev] [PATCH v2 02/12] raw/ifpga_rawdev/base: add irq
> support
> 
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of Rosen Xu
> > Sent: Friday, August 2, 2019 6:49 AM
> > To: dev@dpdk.org
> > Cc: ferruh.yigit@intel.com; tianfei.zhang@intel.com;
> > rosen.xu@intel.com; andy.pei@intel.com; david.lomartire@intel.com;
> > qi.z.zhang@intel.com; xiaolong.ye@intel.com
> > Subject: [dpdk-dev] [PATCH v2 02/12] raw/ifpga_rawdev/base: add irq
> > support
> >
> > From: Tianfei zhang <tianfei.zhang@intel.com>
> >
> > Add irq support for ifpga FME globle error, port error and uint unit.
> > We implmented this feature by vfio interrupt mechanism.
> >
> > Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
> > ---
> >  drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c | 61
> > +++++++++++++++++++++++
> >  drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c   | 22 ++++++++
> >  drivers/raw/ifpga_rawdev/base/ifpga_port.c        | 20 ++++++++
> >  drivers/raw/ifpga_rawdev/base/ifpga_port_error.c  | 21 ++++++++
> >  4 files changed, 124 insertions(+)
> >
> > diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
> > b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
> > index 63c8bcc..6b942e6 100644
> > --- a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
> > +++ b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
> > @@ -3,6 +3,7 @@
> >   */
> >
> >  #include <sys/ioctl.h>
> > +#include <rte_vfio.h>
> >
> >  #include "ifpga_feature_dev.h"
> >
> > @@ -331,3 +332,63 @@ int port_hw_init(struct ifpga_port_hw *port)
> >  	port_hw_uinit(port);
> >  	return ret;
> >  }
> > +
> > +/*
> > + * FIXME: we should get msix vec count during pci enumeration instead
> > +of
> > + * below hardcode value.
> > + */
> > +#define FPGA_MSIX_VEC_COUNT	20
> > +/* irq set buffer length for interrupt */ #define
> > +MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
> > +				sizeof(int) * FPGA_MSIX_VEC_COUNT)
> > +
> > +/* only support msix for now*/
> > +static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
> > +				  unsigned int count, s32 *fds)
> 
> Isn't better to use generic EAL function for the same?

In our PAC N3000 Card, we have 6 PCIe MSI-X vectors, for example:
0~3  for AFU
4    for Port
6    for FME

FME (FPGA Management Engine ) will manage all resources in FPGA, like partition reconfiguration, Power manager, thermal, Error reporting.
Port is a bridge between FME and AFU.
AFU is the accelerator unit which for customers logic.

So, we reserve some MSI-X vectors for end-user/customers to use the AFU, and end-user/customers can use
the AFU for networking acceleration or other acceleration.

The DPDK existing API like rte_intr_enable()->vfio_enable_msix() will bind all of the vectors at the same time and those vectors will register into one evenfd and one interrupt handler function.
That cannot satisfy our design. we hope that, each MSI-X vector bind into VFIO and register the interrupt handler function separately. Because the reserve vectors like
0~3 vectors for AFU, we don't know what exact usage for the end-user/customers in AFU logic, so it had better let them bind VFIO and register interrupt handler themselves.

One suggestion is we expand the vfio_enable_msix() function, let the caller to specify the start vector and the numbers of vectors to bind the VFIO.

static int
vfio_enable_msix(const struct rte_intr_handle *intr_handle, int start, int count) {
    ...
	irq_set->count = count;
	irq_set->start = start;
    ...
	return 0;
}

Re: [dpdk-dev] [PATCH v2 02/12] raw/ifpga_rawdev/base: add irq support

[Jerin Jacob Kollanukkaran]

> -----Original Message-----
> From: Zhang, Tianfei <tianfei.zhang@intel.com>
> Sent: Friday, August 2, 2019 3:36 PM
> To: Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Xu, Rosen
> <rosen.xu@intel.com>; dev@dpdk.org
> Cc: Yigit, Ferruh <ferruh.yigit@intel.com>; Pei, Andy <andy.pei@intel.com>;
> Lomartire, David <david.lomartire@intel.com>; Zhang, Qi Z
> <qi.z.zhang@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>
> Subject: [EXT] RE: [dpdk-dev] [PATCH v2 02/12] raw/ifpga_rawdev/base: add
> irq support
> 
> > > +
> > > +/* only support msix for now*/
> > > +static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int
> vec_start,
> > > +				  unsigned int count, s32 *fds)
> >
> > Isn't better to use generic EAL function for the same?
> 
> In our PAC N3000 Card, we have 6 PCIe MSI-X vectors, for example:
> 0~3  for AFU
> 4    for Port
> 6    for FME
> 
> FME (FPGA Management Engine ) will manage all resources in FPGA, like
> partition reconfiguration, Power manager, thermal, Error reporting.
> Port is a bridge between FME and AFU.
> AFU is the accelerator unit which for customers logic.
> 
> So, we reserve some MSI-X vectors for end-user/customers to use the AFU,
> and end-user/customers can use the AFU for networking acceleration or
> other acceleration.
> 
> The DPDK existing API like rte_intr_enable()->vfio_enable_msix() will bind all
> of the vectors at the same time and those vectors will register into one
> evenfd and one interrupt handler function.
> That cannot satisfy our design. we hope that, each MSI-X vector bind into
> VFIO and register the interrupt handler function separately. Because the
> reserve vectors like
> 0~3 vectors for AFU, we don't know what exact usage for the end-
> user/customers in AFU logic, so it had better let them bind VFIO and register
> interrupt handler themselves.
> 
> One suggestion is we expand the vfio_enable_msix() function, let the caller
> to specify the start vector and the numbers of vectors to bind the VFIO.

Yes, Probably have two variants, vfio_enable_msix() alias to count of 1

> static int
> vfio_enable_msix(const struct rte_intr_handle *intr_handle, int start, int
> count) {
>     ...
> 	irq_set->count = count;
> 	irq_set->start = start;
>     ...
> 	return 0;
> }
> 

[dpdk-dev] [PATCH v2 03/12] raw/ifpga_rawdev/base: clear pending bit

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga_rawdev/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
index 068f52c..a6d3dab 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga_rawdev/base/opae_osdep.h b/drivers/raw/ifpga_rawdev/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_osdep.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 04/12] raw/ifpga_rawdev/base: add SEU error support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_defines.h     |  5 ++-
 drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c   | 43 +++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
index a6d3dab..b496667 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
@@ -257,6 +257,45 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1:0;
+}
+
+static int fme_err_get_seu_emr_low(struct ifpga_fme_hw *fme,
+		u64 *val)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+						  FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
+static int fme_err_get_seu_emr_high(struct ifpga_fme_hw *fme,
+		u64 *val)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+						  FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	*val = readq(&fme_err->seu_emr_h);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +309,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr_low(fme, &prop->data);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr_high(fme, &prop->data);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 05/12] raw/ifpga_rawdev/base: add device tree support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.c | 183 +++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.h |  10 ++
 mk/rte.app.mk                                    |   2 +-
 3 files changed, 194 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
index 9ed10e2..8617173 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(FLASH_BASE + offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index a277c80..c880506 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -319,7 +319,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_pmd_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_pmd_ifpga_rawdev
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_pmd_ifpga_rawdev -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_pmd_ioat_rawdev
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 06/12] raw/ifpga_rawdev/base: align the send buffer for SPI

[Rosen Xu]

From: Tianfei Zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
---
 .../raw/ifpga_rawdev/base/opae_spi_transaction.c   | 40 +++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga_rawdev/base/opae_spi_transaction.c b/drivers/raw/ifpga_rawdev/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga_rawdev/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 07/12] raw/ifpga_rawdev/base: add sensor support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.c | 279 +++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.h |  56 +++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
index 8617173..474941e 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 08/12] raw/ifpga_rawdev/base: introducing sensor APIs

[Rosen Xu]

From: Tianfei Zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_api.c         |  10 ++
 drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga_rawdev/base/ifpga_fme.c         |  21 ++++
 drivers/raw/ifpga_rawdev/base/opae_hw_api.c       | 115 ++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_hw_api.h       |  16 +++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_api.c b/drivers/raw/ifpga_rawdev/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_api.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_fme.c b/drivers/raw/ifpga_rawdev/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_fme.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga_rawdev/base/opae_hw_api.c b/drivers/raw/ifpga_rawdev/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_hw_api.c
+++ b/drivers/raw/ifpga_rawdev/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga_rawdev/base/opae_hw_api.h b/drivers/raw/ifpga_rawdev/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_hw_api.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 09/12] raw/ifpga_rawdev/base: update SEU register definition

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 10/12] raw/ifpga_rawdev: add SEU error handler

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/raw/ifpga_rawdev/ifpga_rawdev.c | 246 ++++++++++++++++++++++++++++++++
 1 file changed, 246 insertions(+)

diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
index fef89e6..7121884 100644
--- a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
@@ -28,6 +28,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -606,6 +608,237 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	struct feature_prop prop;
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", prop.data);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -653,6 +886,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -678,6 +912,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -697,6 +935,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -721,6 +960,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 11/12] raw/ifpga_rawdev: add PCIe BDF devices tree scan

[Rosen Xu]

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/raw/ifpga_rawdev/ifpga_rawdev.c | 553 +++++++++++++++++++++++++++++++-
 drivers/raw/ifpga_rawdev/ifpga_rawdev.h |  16 +
 2 files changed, 562 insertions(+), 7 deletions(-)

diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
index 7121884..16f8387 100644
--- a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_eal_memconfig.h>
@@ -18,7 +20,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -26,6 +28,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -38,6 +41,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR    0x01    /* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE    256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE    4096
+#define RTE_PCI_EXT_CAP_ID(header)    (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)    ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -65,6 +74,493 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	//scan folder
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	//get fpga and fvl
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!sensor)
+			goto fail;
+
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor is invalid value %x\n",
+					value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		//get bridge fd
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		//save previout ECAP_AER+0x08
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		//save previout ECAP_AER+0x14
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		//set ECAP_AER+0x08 to 0xFFFFFFFF
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		//set ECAP_AER+0x14 to 0xFFFFFFFF
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		//read current ECAP_AER+0x08
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		//read current ECAP_AER+0x14
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				printf(">>>>>>Check Device %s\n",
+					ifpga_rdev->rawdev->name);
+				ret = set_surprise_link_check_aer(ifpga_rdev);
+				if (ret == 1)
+					gsd_enable = 1;
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(1000 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR("Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -373,8 +869,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -644,7 +1141,6 @@
 static int
 fme_err_read_seu_emr(struct opae_manager *mgr)
 {
-	struct feature_prop prop;
 	u64 val;
 	int ret;
 
@@ -658,7 +1154,7 @@
 	if (ret)
 		return -EINVAL;
 
-	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", prop.data);
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
 
 	return 0;
 }
@@ -844,6 +1340,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -857,7 +1354,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -871,6 +1368,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -989,6 +1494,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1020,13 +1526,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	strlcpy(*(char **)extra_args, value, size);
+
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1040,7 +1565,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1067,6 +1593,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.h b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 12/12] net/ipn3ke: remove configuration for i40e port bonding

[Rosen Xu]

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/net/ipn3ke/Makefile             |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c      | 289 ++++----------------------------
 drivers/net/ipn3ke/ipn3ke_representor.c |   7 +-
 3 files changed, 43 insertions(+), 255 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..5478fd9 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga_rawdev
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index c226d63..363a5f1 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -241,7 +242,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -259,6 +261,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -323,9 +331,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -376,7 +381,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -403,7 +412,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -415,6 +429,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -422,6 +452,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -467,254 +498,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index 8300cc3..a4ee460 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2906,8 +2907,10 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		i40e_set_switch_dev(rpst->i40e_pf_eth, rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v2 00/12] Add PCIe AER disable and IRQ support for ipn3ke

[Jerin Jacob Kollanukkaran]

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Rosen Xu
> Sent: Friday, August 2, 2019 6:49 AM
> To: dev@dpdk.org
> Cc: ferruh.yigit@intel.com; tianfei.zhang@intel.com; rosen.xu@intel.com;
> andy.pei@intel.com; david.lomartire@intel.com; qi.z.zhang@intel.com;
> xiaolong.ye@intel.com
> Subject: [dpdk-dev] [PATCH v2 00/12] Add PCIe AER disable and IRQ support
> for ipn3ke
> 
> This patch set adds PCIe AER disable and IRQ support for ipn3ke.
> Disable PCIe AER is very useful when FPGA reload. IRQ is used very widely in
> interrupt process.

Shouldn't it better to have common code in PCI subsystem to disable PCIe AER etc,
So that other drivers can be used in future.

> 
> For ipn3ke is connect to CPU with PCIe switch, driver needs to scan all PCIe

Do we need a special PCIe switch for this? Or Generic PCIe switch would do?

> devices of ipn3ke, it also can get all i40e of card, so ipn3ke driver doesn't
> need to take some configuration of i40e.

Is communication between i40e and ipn3ke proprietary scheme?
Who is the PCIe bus master here? Ipn3ke or i40e?

Re: [dpdk-dev] [PATCH v2 00/12] Add PCIe AER disable and IRQ support for ipn3ke

[Xu, Rosen]

Hi,

> -----Original Message-----
> From: Jerin Jacob Kollanukkaran [mailto:jerinj@marvell.com]
> Sent: Friday, August 02, 2019 12:15
> To: Xu, Rosen <rosen.xu@intel.com>; dev@dpdk.org
> Cc: Yigit, Ferruh <ferruh.yigit@intel.com>; Zhang, Tianfei
> <tianfei.zhang@intel.com>; Pei, Andy <andy.pei@intel.com>; Lomartire,
> David <david.lomartire@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Ye,
> Xiaolong <xiaolong.ye@intel.com>
> Subject: RE: [dpdk-dev] [PATCH v2 00/12] Add PCIe AER disable and IRQ
> support for ipn3ke
> 
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of Rosen Xu
> > Sent: Friday, August 2, 2019 6:49 AM
> > To: dev@dpdk.org
> > Cc: ferruh.yigit@intel.com; tianfei.zhang@intel.com;
> > rosen.xu@intel.com; andy.pei@intel.com; david.lomartire@intel.com;
> > qi.z.zhang@intel.com; xiaolong.ye@intel.com
> > Subject: [dpdk-dev] [PATCH v2 00/12] Add PCIe AER disable and IRQ
> > support for ipn3ke
> >
> > This patch set adds PCIe AER disable and IRQ support for ipn3ke.
> > Disable PCIe AER is very useful when FPGA reload. IRQ is used very
> > widely in interrupt process.
> 
> Shouldn't it better to have common code in PCI subsystem to disable PCIe
> AER etc, So that other drivers can be used in future.

That's a good proposal. But there's something special in IPN3KE.
In IPN3KE, one Intel A10 FPGA and two I40e are connected to CPU with PCIe
switch chip, there are some errors when PCIe switch chip bonding to VFIO,
in our design, we access PCIe configure space with pread/pwrite.
For AER disable, we need access PCIe switch chip configuration space.

> >
> > For ipn3ke is connect to CPU with PCIe switch, driver needs to scan
> > all PCIe
> 
> Do we need a special PCIe switch for this? Or Generic PCIe switch would do?

It's hardware specific.
 
> > devices of ipn3ke, it also can get all i40e of card, so ipn3ke driver
> > doesn't need to take some configuration of i40e.
> 
> Is communication between i40e and ipn3ke proprietary scheme?

Yes.

> Who is the PCIe bus master here? Ipn3ke or i40e?

From DPDK point of view, there are 3 PCIe devices in DPDK  one Intel A10 FPGA and two I40e.
No master.

> 
> 

[dpdk-dev] [PATCH v3 00/13] Add PCIe AER disable and IRQ support for ipn3ke

[Rosen Xu]

This patch set adds PCIe AER disable and IRQ support for ipn3ke.
Disable PCIe AER is very useful when FPGA reload. IRQ is used
very widely in interrupt process.

For ipn3ke is connect to CPU with PCIe switch, driver needs to
scan all PCIe devices of ipn3ke, it also can get all i40e of card,
so ipn3ke driver doesn't need to take some configuration of i40e.

v3 updates:
===========
 - Add FPGA network side port MTU configuration

v2 updates:
===========
 - Add AUX feature support

Rosen Xu (4):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga_rawdev: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding
  net/ipn3ke: add FPGA network side port MTU configuration

Tianfei Zhang (2):
  raw/ifpga_rawdev/base: align the send buffer for SPI
  raw/ifpga_rawdev/base: introducing sensor APIs

Tianfei zhang (7):
  raw/ifpga_rawdev/base: add irq support
  raw/ifpga_rawdev/base: clear pending bit
  raw/ifpga_rawdev/base: add SEU error support
  raw/ifpga_rawdev/base: add device tree support
  raw/ifpga_rawdev/base: add sensor support
  raw/ifpga_rawdev/base: update SEU register definition
  raw/ifpga_rawdev: add SEU error handler

 drivers/net/i40e/base/i40e_type.h                  |   3 +
 drivers/net/i40e/i40e_ethdev.c                     |  34 +-
 drivers/net/i40e/rte_pmd_i40e.h                    |   4 +
 drivers/net/ipn3ke/Makefile                        |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c                 | 297 ++------
 drivers/net/ipn3ke/ipn3ke_ethdev.h                 |  55 ++
 drivers/net/ipn3ke/ipn3ke_representor.c            |   7 +-
 drivers/raw/ifpga_rawdev/base/ifpga_api.c          |  10 +
 drivers/raw/ifpga_rawdev/base/ifpga_defines.h      |  18 +-
 drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c  |  61 ++
 drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h  |   3 +
 drivers/raw/ifpga_rawdev/base/ifpga_fme.c          |  21 +
 drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c    |  69 +-
 drivers/raw/ifpga_rawdev/base/ifpga_port.c         |  20 +
 drivers/raw/ifpga_rawdev/base/ifpga_port_error.c   |  21 +
 drivers/raw/ifpga_rawdev/base/opae_hw_api.c        | 115 +++
 drivers/raw/ifpga_rawdev/base/opae_hw_api.h        |  16 +
 drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h  |   2 +
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.c   | 462 ++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.h   |  66 ++
 drivers/raw/ifpga_rawdev/base/opae_osdep.h         |   7 +-
 .../raw/ifpga_rawdev/base/opae_spi_transaction.c   |  40 +-
 drivers/raw/ifpga_rawdev/ifpga_rawdev.c            | 793 ++++++++++++++++++++-
 drivers/raw/ifpga_rawdev/ifpga_rawdev.h            |  16 +
 mk/rte.app.mk                                      |   2 +-
 25 files changed, 1866 insertions(+), 278 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v3 01/13] net/i40e: i40e support ipn3ke FPGA port bonding

[Rosen Xu]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/net/i40e/base/i40e_type.h |  3 +++
 drivers/net/i40e/i40e_ethdev.c    | 34 ++++++++++++++++++++++++++++++++--
 drivers/net/i40e/rte_pmd_i40e.h   |  4 ++++
 3 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..a4d46d8 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	//switch device
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 4e40b7a..e981256 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1312,6 +1312,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	//Update switch device pointer
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2782,6 +2785,20 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	}
 }
 
+void
+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
+struct rte_eth_dev *switch_dev)
+{
+	struct i40e_hw *hw;
+
+	if (!i40e_dev)
+		return;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+
+	hw->switch_dev = switch_dev;
+}
+
 int
 i40e_dev_link_update(struct rte_eth_dev *dev,
 		     int wait_to_complete)
@@ -2790,6 +2807,7 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	struct rte_eth_link link;
 	bool enable_lse = dev->data->dev_conf.intr_conf.lsc ? true : false;
 	int ret;
+	struct rte_eth_dev *switch_ethdev;
 
 	memset(&link, 0, sizeof(link));
 
@@ -2803,6 +2821,18 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	switch_ethdev = hw->switch_dev;
+	if (switch_ethdev) {
+		rte_eth_linkstatus_get(switch_ethdev, &link);
+		printf(">>>>>>>>>>>>>i40e_update_link 5 link.link_status %d\n",
+			link.link_status);
+	} else {
+		link.link_duplex = ETH_LINK_FULL_DUPLEX;
+		link.link_autoneg = ETH_LINK_SPEED_FIXED;
+		link.link_speed = ETH_SPEED_NUM_25G;
+		link.link_status = 0;
+	}
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
@@ -12541,7 +12571,7 @@ struct i40e_customized_pctype*
  *	b.	Old_filter = 10 (Stag_Inner_Vlan)
  *	c.	New_filter = 0x10
  *	d.	TR bit = 0xff (optional, not used here)
- *	e.	Buffer – 2 entries:
+ *	e.	Buffer - 2 entries:
  *		i.	Byte 0 = 8 (outer vlan FV index).
  *			Byte 1 = 0 (rsv)
  *			Byte 2-3 = 0x0fff
@@ -12555,7 +12585,7 @@ struct i40e_customized_pctype*
  *	a.	Valid_flags.replace_cloud = 1
  *	b.	Old_filter = 1 (instead of outer IP)
  *	c.	New_filter = 0x10
- *	d.	Buffer – 2 entries:
+ *	d.	Buffer - 2 entries:
  *		i.	Byte 0 = 0x80 | 7 (valid | Stag).
  *			Byte 1-3 = 0 (rsv)
  *		ii.	Byte 8 = 0x80 | 0x10 (valid | new l1 filter step1)
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..9d77c85 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,8 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+void
+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
+struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 02/13] raw/ifpga_rawdev/base: add irq support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME globle error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c | 61 +++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c   | 22 ++++++++
 drivers/raw/ifpga_rawdev/base/ifpga_port.c        | 20 ++++++++
 drivers/raw/ifpga_rawdev/base/ifpga_port_error.c  | 21 ++++++++
 4 files changed, 124 insertions(+)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
index 63c8bcc..6b942e6 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,63 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+/*
+ * FIXME: we should get msix vec count during pci enumeration instead of
+ * below hardcode value.
+ */
+#define FPGA_MSIX_VEC_COUNT	20
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	irq_set->count = count;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
+
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
index 3794564..068f52c 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
@@ -373,9 +373,31 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set =
+			(struct fpga_fme_err_irq_set *)irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	spinlock_lock(&fme->lock);
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ)) {
+		spinlock_unlock(&fme->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_port.c b/drivers/raw/ifpga_rawdev/base/ifpga_port.c
index 6c41164..56b04a6 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_port.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_port.c
@@ -384,9 +384,29 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	spinlock_lock(&port->lock);
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ)) {
+		spinlock_unlock(&port->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c b/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c
index 138284e..8aef7d7 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c
@@ -136,9 +136,30 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	spinlock_lock(&port->lock);
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ)) {
+		spinlock_unlock(&port->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 03/13] raw/ifpga_rawdev/base: clear pending bit

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga_rawdev/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
index 068f52c..a6d3dab 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga_rawdev/base/opae_osdep.h b/drivers/raw/ifpga_rawdev/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_osdep.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 04/13] raw/ifpga_rawdev/base: add SEU error support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_defines.h     |  5 ++-
 drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c   | 43 +++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
index a6d3dab..b496667 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
@@ -257,6 +257,45 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1:0;
+}
+
+static int fme_err_get_seu_emr_low(struct ifpga_fme_hw *fme,
+		u64 *val)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+						  FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
+static int fme_err_get_seu_emr_high(struct ifpga_fme_hw *fme,
+		u64 *val)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+						  FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	*val = readq(&fme_err->seu_emr_h);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +309,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr_low(fme, &prop->data);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr_high(fme, &prop->data);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 05/13] raw/ifpga_rawdev/base: add device tree support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.c | 183 +++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.h |  10 ++
 mk/rte.app.mk                                    |   2 +-
 3 files changed, 194 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index a277c80..c880506 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -319,7 +319,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_pmd_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_pmd_ifpga_rawdev
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_pmd_ifpga_rawdev -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_pmd_ioat_rawdev
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 06/13] raw/ifpga_rawdev/base: align the send buffer for SPI

[Rosen Xu]

From: Tianfei Zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
---
 .../raw/ifpga_rawdev/base/opae_spi_transaction.c   | 40 +++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga_rawdev/base/opae_spi_transaction.c b/drivers/raw/ifpga_rawdev/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga_rawdev/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 07/13] raw/ifpga_rawdev/base: add sensor support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.c | 279 +++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.h |  56 +++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 08/13] raw/ifpga_rawdev/base: introducing sensor APIs

[Rosen Xu]

From: Tianfei Zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_api.c         |  10 ++
 drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga_rawdev/base/ifpga_fme.c         |  21 ++++
 drivers/raw/ifpga_rawdev/base/opae_hw_api.c       | 115 ++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_hw_api.h       |  16 +++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_api.c b/drivers/raw/ifpga_rawdev/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_api.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_fme.c b/drivers/raw/ifpga_rawdev/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_fme.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga_rawdev/base/opae_hw_api.c b/drivers/raw/ifpga_rawdev/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_hw_api.c
+++ b/drivers/raw/ifpga_rawdev/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga_rawdev/base/opae_hw_api.h b/drivers/raw/ifpga_rawdev/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_hw_api.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 09/13] raw/ifpga_rawdev/base: update SEU register definition

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 10/13] raw/ifpga_rawdev: add SEU error handler

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/raw/ifpga_rawdev/ifpga_rawdev.c | 246 ++++++++++++++++++++++++++++++++
 1 file changed, 246 insertions(+)

diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
index fef89e6..7121884 100644
--- a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
@@ -28,6 +28,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -606,6 +608,237 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	struct feature_prop prop;
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", prop.data);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -653,6 +886,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -678,6 +912,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -697,6 +935,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -721,6 +960,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 11/13] raw/ifpga_rawdev: add PCIe BDF devices tree scan

[Rosen Xu]

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/raw/ifpga_rawdev/ifpga_rawdev.c | 551 +++++++++++++++++++++++++++++++-
 drivers/raw/ifpga_rawdev/ifpga_rawdev.h |  16 +
 2 files changed, 560 insertions(+), 7 deletions(-)

diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
index 7121884..abaefb1 100644
--- a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_eal_memconfig.h>
@@ -18,7 +20,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -26,6 +28,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -38,6 +41,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR    0x01    /* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE    256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE    4096
+#define RTE_PCI_EXT_CAP_ID(header)    (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)    ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -65,6 +74,491 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	//scan folder
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	//get fpga and fvl
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!sensor)
+			goto fail;
+
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor is invalid value %x\n",
+					value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		//get bridge fd
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		//save previout ECAP_AER+0x08
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		//save previout ECAP_AER+0x14
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		//set ECAP_AER+0x08 to 0xFFFFFFFF
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		//set ECAP_AER+0x14 to 0xFFFFFFFF
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		//read current ECAP_AER+0x08
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		//read current ECAP_AER+0x14
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev);
+				if (ret == 1)
+					gsd_enable = 1;
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100000 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR("Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -373,8 +867,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -644,7 +1139,6 @@
 static int
 fme_err_read_seu_emr(struct opae_manager *mgr)
 {
-	struct feature_prop prop;
 	u64 val;
 	int ret;
 
@@ -658,7 +1152,7 @@
 	if (ret)
 		return -EINVAL;
 
-	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", prop.data);
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
 
 	return 0;
 }
@@ -844,6 +1338,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -857,7 +1352,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -871,6 +1366,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -989,6 +1492,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1020,13 +1524,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	strlcpy(*(char **)extra_args, value, size);
+
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1040,7 +1563,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1067,6 +1591,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.h b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 12/13] net/ipn3ke: remove configuration for i40e port bonding

[Rosen Xu]

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/net/ipn3ke/Makefile             |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c      | 289 ++++----------------------------
 drivers/net/ipn3ke/ipn3ke_representor.c |   7 +-
 3 files changed, 43 insertions(+), 255 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..5478fd9 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga_rawdev
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index c226d63..363a5f1 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -241,7 +242,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -259,6 +261,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -323,9 +331,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -376,7 +381,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -403,7 +412,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -415,6 +429,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -422,6 +452,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -467,254 +498,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index 8300cc3..a4ee460 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2906,8 +2907,10 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		i40e_set_switch_dev(rpst->i40e_pf_eth, rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 13/13] net/ipn3ke: add FPGA network side port MTU configuration

[Rosen Xu]

Add FPGA network side port MTU configuration in initialization.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/net/ipn3ke/ipn3ke_ethdev.c |  8 ++++++
 drivers/net/ipn3ke/ipn3ke_ethdev.h | 55 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)

diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index 363a5f1..7e7fa25 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -292,6 +292,10 @@
 
 			/* Clear line RX statistics counters */
 			ipn3ke_xmac_rx_clr_10G_stcs(hw, i, 0);
+
+			/* set mtu to max */
+			ipn3ke_10G_mtu_setup(hw, i, 0);
+			ipn3ke_10G_mtu_setup(hw, i, 1);
 		}
 	} else if (hw->retimer.mac_type ==
 			IFPGA_RAWDEV_RETIMER_MAC_TYPE_25GE_25GAUI) {
@@ -308,6 +312,10 @@
 
 			/* Clear line side RX statistics counters */
 			ipn3ke_xmac_rx_clr_25G_stcs(hw, i, 0);
+
+			/* set mtu to max */
+			ipn3ke_25G_mtu_setup(hw, i, 0);
+			ipn3ke_25G_mtu_setup(hw, i, 1);
 		}
 	}
 
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.h b/drivers/net/ipn3ke/ipn3ke_ethdev.h
index c7b336b..b04e5d3 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.h
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.h
@@ -654,6 +654,25 @@ static inline void _ipn3ke_indrct_write(struct ipn3ke_hw *hw,
 #define IPN3KE_MAC_RX_FRAME_MAXLENGTH_MASK \
 	IPN3KE_MASK(0xFFFF, IPN3KE_MAC_RX_FRAME_MAXLENGTH_SHIFT)
 
+/* Additional Feature Register */
+#define ADD_PHY_CTRL		0x0
+#define PHY_RESET		BIT(0)
+/* registers for 25G/40G mac */
+#define MAC_CONFIG	0x310
+#define MAC_RESET_MASK	GENMASK(2, 0)
+
+#define IPN3KE_MAX_MTU			0xffff
+
+#define IPN3KE_25G_PHY_PMA_SLOOP		0x313
+#define IPN3KE_25G_TX_FLOW_CTRL		0x640
+#define IPN3KE_25G_MAX_TX_SIZE_CONFIG	0x407
+#define IPN3KE_25G_MAX_RX_SIZE_CONFIG	0x506
+
+#define IPN3KE_10G_TX_PAUSE_FRAME_QUANTA	0x42
+#define IPN3KE_10G_TX_PAUSE_FRAME_HOLDOFF	0x43
+#define IPN3KE_10G_TX_FRAME_MAXLENGTH	0x2c
+#define IPN3KE_10G_RX_FRAME_MAXLENGTH	0xae
+
 #define IPN3KE_REGISTER_WIDTH                                        32
 
 /*Bits[2:0]: Configuration of TX statistics counters:
@@ -1076,4 +1095,40 @@ static inline void ipn3ke_xmac_smac_ovd_dis(struct ipn3ke_hw *hw,
 					eth_group_sel);
 }
 
+static inline void ipn3ke_10G_mtu_setup
+(struct ipn3ke_hw *hw, uint32_t mac_num, uint32_t eth_group_sel)
+{
+	uint32_t tmp = IPN3KE_MAC_FRAME_SIZE_MAX;
+
+	(*hw->f_mac_write)(hw,
+					tmp,
+					IPN3KE_10G_TX_FRAME_MAXLENGTH,
+					mac_num,
+					eth_group_sel);
+
+	(*hw->f_mac_write)(hw,
+					tmp,
+					IPN3KE_10G_RX_FRAME_MAXLENGTH,
+					mac_num,
+					eth_group_sel);
+}
+
+static inline void ipn3ke_25G_mtu_setup
+(struct ipn3ke_hw *hw, uint32_t mac_num, uint32_t eth_group_sel)
+{
+	uint32_t tmp = IPN3KE_MAC_FRAME_SIZE_MAX;
+
+	(*hw->f_mac_write)(hw,
+					tmp,
+					IPN3KE_25G_MAX_TX_SIZE_CONFIG,
+					mac_num,
+					eth_group_sel);
+
+	(*hw->f_mac_write)(hw,
+					tmp,
+					IPN3KE_25G_MAX_RX_SIZE_CONFIG,
+					mac_num,
+					eth_group_sel);
+}
+
 #endif /* _IPN3KE_ETHDEV_H_ */
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v3 13/13] net/ipn3ke: add FPGA network side port MTU configuration

[Pei, Andy]

This patch set both line side and nic side MTU when Hardware init, and init to IPN3KE_MAC_FRAME_SIZE_MAX.

-----Original Message-----
From: Xu, Rosen 
Sent: Thursday, August 8, 2019 4:46 PM
To: dev@dpdk.org
Cc: Yigit, Ferruh <ferruh.yigit@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>; Xu, Rosen <rosen.xu@intel.com>; Pei, Andy <andy.pei@intel.com>; Lomartire, David <david.lomartire@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>
Subject: [PATCH v3 13/13] net/ipn3ke: add FPGA network side port MTU configuration

Add FPGA network side port MTU configuration in initialization.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/net/ipn3ke/ipn3ke_ethdev.c |  8 ++++++  drivers/net/ipn3ke/ipn3ke_ethdev.h | 55 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)

diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index 363a5f1..7e7fa25 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -292,6 +292,10 @@
 
 			/* Clear line RX statistics counters */
 			ipn3ke_xmac_rx_clr_10G_stcs(hw, i, 0);
+
+			/* set mtu to max */
+			ipn3ke_10G_mtu_setup(hw, i, 0);
+			ipn3ke_10G_mtu_setup(hw, i, 1);
 		}
 	} else if (hw->retimer.mac_type ==
 			IFPGA_RAWDEV_RETIMER_MAC_TYPE_25GE_25GAUI) { @@ -308,6 +312,10 @@
 
 			/* Clear line side RX statistics counters */
 			ipn3ke_xmac_rx_clr_25G_stcs(hw, i, 0);
+
+			/* set mtu to max */
+			ipn3ke_25G_mtu_setup(hw, i, 0);
+			ipn3ke_25G_mtu_setup(hw, i, 1);
 		}
 	}
 
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.h b/drivers/net/ipn3ke/ipn3ke_ethdev.h
index c7b336b..b04e5d3 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.h
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.h
@@ -654,6 +654,25 @@ static inline void _ipn3ke_indrct_write(struct ipn3ke_hw *hw,  #define IPN3KE_MAC_RX_FRAME_MAXLENGTH_MASK \
 	IPN3KE_MASK(0xFFFF, IPN3KE_MAC_RX_FRAME_MAXLENGTH_SHIFT)
 
+/* Additional Feature Register */
+#define ADD_PHY_CTRL		0x0
+#define PHY_RESET		BIT(0)
+/* registers for 25G/40G mac */
+#define MAC_CONFIG	0x310
+#define MAC_RESET_MASK	GENMASK(2, 0)
+
+#define IPN3KE_MAX_MTU			0xffff
+
+#define IPN3KE_25G_PHY_PMA_SLOOP		0x313
+#define IPN3KE_25G_TX_FLOW_CTRL		0x640
+#define IPN3KE_25G_MAX_TX_SIZE_CONFIG	0x407
+#define IPN3KE_25G_MAX_RX_SIZE_CONFIG	0x506
+
+#define IPN3KE_10G_TX_PAUSE_FRAME_QUANTA	0x42
+#define IPN3KE_10G_TX_PAUSE_FRAME_HOLDOFF	0x43
+#define IPN3KE_10G_TX_FRAME_MAXLENGTH	0x2c
+#define IPN3KE_10G_RX_FRAME_MAXLENGTH	0xae
+
 #define IPN3KE_REGISTER_WIDTH                                        32
 
 /*Bits[2:0]: Configuration of TX statistics counters:
@@ -1076,4 +1095,40 @@ static inline void ipn3ke_xmac_smac_ovd_dis(struct ipn3ke_hw *hw,
 					eth_group_sel);
 }
 
+static inline void ipn3ke_10G_mtu_setup (struct ipn3ke_hw *hw, uint32_t 
+mac_num, uint32_t eth_group_sel) {
+	uint32_t tmp = IPN3KE_MAC_FRAME_SIZE_MAX;
+
+	(*hw->f_mac_write)(hw,
+					tmp,
+					IPN3KE_10G_TX_FRAME_MAXLENGTH,
+					mac_num,
+					eth_group_sel);
+
+	(*hw->f_mac_write)(hw,
+					tmp,
+					IPN3KE_10G_RX_FRAME_MAXLENGTH,
+					mac_num,
+					eth_group_sel);
+}
+
+static inline void ipn3ke_25G_mtu_setup (struct ipn3ke_hw *hw, uint32_t 
+mac_num, uint32_t eth_group_sel) {
+	uint32_t tmp = IPN3KE_MAC_FRAME_SIZE_MAX;
+
+	(*hw->f_mac_write)(hw,
+					tmp,
+					IPN3KE_25G_MAX_TX_SIZE_CONFIG,
+					mac_num,
+					eth_group_sel);
+
+	(*hw->f_mac_write)(hw,
+					tmp,
+					IPN3KE_25G_MAX_RX_SIZE_CONFIG,
+					mac_num,
+					eth_group_sel);
+}
+
 #endif /* _IPN3KE_ETHDEV_H_ */
--
1.8.3.1

[dpdk-dev] [PATCH v4 00/12] Add PCIe AER disable and IRQ support for ipn3ke

[Andy Pei]

This patch set adds PCIe AER disable and IRQ support for ipn3ke.
Disable PCIe AER is very useful when FPGA reload. IRQ is used very
widely in interrupt process.

For ipn3ke is connect to CPU with PCIe switch, driver needs to scan
all PCIe devices of ipn3ke, it also can get all i40e of card, so
ipn3ke driver doesn't need to take some configuration of i40e.

v4 updates:
==========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support

Rosen Xu (3):
 net/i40e: i40e support ipn3ke FPGA port bonding
 raw/ifpga: add PCIe BDF devices tree scan
 net/ipn3ke: remove configuration for i40e port bonding

Tianfei Zhang (2):
 raw/ifpga/base: align the send buffer for SPI
 raw/ifpga/base: introducing sensor APIs

Tianfei zhang (7):
 raw/ifpga/base: add irq support
 raw/ifpga/base: clear pending bit
 raw/ifpga/base: add SEU error support
 raw/ifpga/base: add device tree support
 raw/ifpga/base: add sensor support
 raw/ifpga/base: update SEU register definition
 raw/ifpga: add SEU error handler


 drivers/net/i40e/base/i40e_type.h             |   3 +
 drivers/net/i40e/i40e_ethdev.c                |  32 +-
 drivers/net/i40e/rte_pmd_i40e.h               |   4 +
 drivers/net/ipn3ke/Makefile                   |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 289 ++--------
 drivers/net/ipn3ke/ipn3ke_representor.c       |   7 +-
 drivers/raw/ifpga/base/ifpga_api.c            |  10 +
 drivers/raw/ifpga/base/ifpga_defines.h        |  18 +-
 drivers/raw/ifpga/base/ifpga_feature_dev.c    |  61 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c            |  21 +
 drivers/raw/ifpga/base/ifpga_fme_error.c      |  69 ++-
 drivers/raw/ifpga/base/ifpga_port.c           |  20 +
 drivers/raw/ifpga/base/ifpga_port_error.c     |  21 +
 drivers/raw/ifpga/base/opae_hw_api.c          | 115 ++++
 drivers/raw/ifpga/base/opae_hw_api.h          |  16 +
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c     | 462 +++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h     |  66 +++
 drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  40 +-
 drivers/raw/ifpga/ifpga_rawdev.c              | 791 +++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h              |  16 +
 mk/rte.app.mk                                 |   2 +-
 24 files changed, 1799 insertions(+), 278 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v4 01/12] net/i40e: i40e support ipn3ke FPGA port bonding

[Andy Pei]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=y, Size: 3676 bytes --]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h |  3 +++
 drivers/net/i40e/i40e_ethdev.c    | 32 ++++++++++++++++++++++++++++++--
 drivers/net/i40e/rte_pmd_i40e.h   |  4 ++++
 3 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..15f26ad 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 4e40b7a..f95c947 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1312,6 +1312,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Update switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2782,6 +2785,20 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	}
 }
 
+void
+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
+struct rte_eth_dev *switch_dev)
+{
+	struct i40e_hw *hw;
+
+	if (!i40e_dev)
+		return;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+
+	hw->switch_dev = switch_dev;
+}
+
 int
 i40e_dev_link_update(struct rte_eth_dev *dev,
 		     int wait_to_complete)
@@ -2790,6 +2807,7 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	struct rte_eth_link link;
 	bool enable_lse = dev->data->dev_conf.intr_conf.lsc ? true : false;
 	int ret;
+	struct rte_eth_dev *switch_ethdev;
 
 	memset(&link, 0, sizeof(link));
 
@@ -2803,6 +2821,16 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	switch_ethdev = hw->switch_dev;
+	if (switch_ethdev) {
+		rte_eth_linkstatus_get(switch_ethdev, &link);
+	} else {
+		link.link_duplex = ETH_LINK_FULL_DUPLEX;
+		link.link_autoneg = ETH_LINK_SPEED_FIXED;
+		link.link_speed = ETH_SPEED_NUM_25G;
+		link.link_status = 0;
+	}
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
@@ -12541,7 +12569,7 @@ struct i40e_customized_pctype*
  *	b.	Old_filter = 10 (Stag_Inner_Vlan)
  *	c.	New_filter = 0x10
  *	d.	TR bit = 0xff (optional, not used here)
- *	e.	Buffer – 2 entries:
+ *	e.	Buffer - 2 entries:
  *		i.	Byte 0 = 8 (outer vlan FV index).
  *			Byte 1 = 0 (rsv)
  *			Byte 2-3 = 0x0fff
@@ -12555,7 +12583,7 @@ struct i40e_customized_pctype*
  *	a.	Valid_flags.replace_cloud = 1
  *	b.	Old_filter = 1 (instead of outer IP)
  *	c.	New_filter = 0x10
- *	d.	Buffer – 2 entries:
+ *	d.	Buffer - 2 entries:
  *		i.	Byte 0 = 0x80 | 7 (valid | Stag).
  *			Byte 1-3 = 0 (rsv)
  *		ii.	Byte 8 = 0x80 | 0x10 (valid | new l1 filter step1)
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..9d77c85 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,8 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+void
+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
+struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 00/17] Add PCIe AER disable and IRQ support for ipn3ke

[Andy Pei]

s patch set adds PCIe AER disable and IRQ support for ipn3ke.
Disable PCIe AER is very useful when FPGA reload. IRQ is used very
widely in interrupt process.

For ipn3ke is connect to CPU with PCIe switch, driver needs to scan
all PCIe devices of ipn3ke, it also can get all i40e of card, so
ipn3ke driver doesn't need to take some configuration of i40e.

v5 update:
=========
- add lightweight fpga image support. in lightweight fpga image mode, ipn3ke 
  representor will not be probed.

v4 updates:
==========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support


Andy Pei (13):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding
  raw/ifpga: add lightweight fpga image support

Tianfei Zhang (2):
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors

Tianfei zhang (2):
  raw/ifpga/base: add secure support
  raw/ifpga/base: add new API get board info

 drivers/net/i40e/base/i40e_type.h             |   3 +
 drivers/net/i40e/i40e_ethdev.c                |  20 +
 drivers/net/i40e/rte_pmd_i40e.h               |   4 +
 drivers/net/ipn3ke/Makefile                   |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 289 ++-------
 drivers/net/ipn3ke/ipn3ke_representor.c       |   7 +-
 drivers/raw/ifpga/base/ifpga_api.c            |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h        |  75 ++-
 drivers/raw/ifpga/base/ifpga_feature_dev.c    |  60 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c            | 138 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c      |  89 ++-
 drivers/raw/ifpga/base/ifpga_hw.h             |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c           |  20 +
 drivers/raw/ifpga/base/ifpga_port_error.c     |  21 +
 drivers/raw/ifpga/base/opae_hw_api.c          | 135 ++++
 drivers/raw/ifpga/base/opae_hw_api.h          |  21 +
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c     | 568 ++++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h     | 146 ++++-
 drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
 drivers/raw/ifpga/base/opae_spi.h             |  23 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  40 +-
 drivers/raw/ifpga/ifpga_rawdev.c              | 851 +++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h              |  16 +
 mk/rte.app.mk                                 |   2 +-
 26 files changed, 2187 insertions(+), 378 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v5 01/17] net/i40e: i40e support ipn3ke FPGA port bonding

[Andy Pei]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h |  3 +++
 drivers/net/i40e/i40e_ethdev.c    | 20 ++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h   |  4 ++++
 3 files changed, 27 insertions(+)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 4e40b7a..6c4b1e8 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1312,6 +1312,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2782,6 +2785,20 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	}
 }
 
+void
+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
+struct rte_eth_dev *switch_dev)
+{
+	struct i40e_hw *hw;
+
+	if (!i40e_dev)
+		return;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+
+	hw->switch_dev = switch_dev;
+}
+
 int
 i40e_dev_link_update(struct rte_eth_dev *dev,
 		     int wait_to_complete)
@@ -2803,6 +2820,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..9d77c85 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,8 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+void
+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
+struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 00/17] add PCIe AER disable and IRQ support for ipn3ke

[Andy Pei]

This patch set adds PCIe AER disable and IRQ support for ipn3ke.
Disable PCIe AER is very useful when FPGA reload. IRQ is used very
widely in interrupt process.

For ipn3ke is connect to CPU with PCIe switch, driver needs to scan
all PCIe devices of ipn3ke, it also can get all i40e of card, so
ipn3ke driver doesn't need to take some configuration of i40e.

v6 update:
========
- correct author information.
- correct typo in commit message and remove Gerrit Change-Id's before
  submitting upstream

v5 update:
=========
- add lightweight fpga image support. in lightweight fpga image mode,
  ipn3ke representor will not be probed.

v4 updates:
==========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support


Andy Pei (2):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga: add lightweight fpga image support

Rosen Xu (3):
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei Zhang (2):
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors

Tianfei zhang (10):
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga/base: add secure support
  raw/ifpga/base: add new API get board info

 drivers/net/i40e/base/i40e_type.h             |   3 +
 drivers/net/i40e/i40e_ethdev.c                |  20 +
 drivers/net/i40e/rte_pmd_i40e.h               |   4 +
 drivers/net/ipn3ke/Makefile                   |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 289 ++-------
 drivers/net/ipn3ke/ipn3ke_representor.c       |   7 +-
 drivers/raw/ifpga/base/ifpga_api.c            |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h        |  75 ++-
 drivers/raw/ifpga/base/ifpga_feature_dev.c    |  60 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c            | 138 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c      |  89 ++-
 drivers/raw/ifpga/base/ifpga_hw.h             |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c           |  20 +
 drivers/raw/ifpga/base/ifpga_port_error.c     |  21 +
 drivers/raw/ifpga/base/opae_hw_api.c          | 135 ++++
 drivers/raw/ifpga/base/opae_hw_api.h          |  21 +
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c     | 568 ++++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h     | 146 ++++-
 drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
 drivers/raw/ifpga/base/opae_spi.h             |  23 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  40 +-
 drivers/raw/ifpga/ifpga_rawdev.c              | 851 +++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h              |  16 +
 mk/rte.app.mk                                 |   2 +-
 26 files changed, 2187 insertions(+), 378 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v6 01/17] net/i40e: i40e support ipn3ke FPGA port bonding

[Andy Pei]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h |  3 +++
 drivers/net/i40e/i40e_ethdev.c    | 20 ++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h   |  4 ++++
 3 files changed, 27 insertions(+)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 4e40b7a..6c4b1e8 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1312,6 +1312,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2782,6 +2785,20 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	}
 }
 
+void
+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
+struct rte_eth_dev *switch_dev)
+{
+	struct i40e_hw *hw;
+
+	if (!i40e_dev)
+		return;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+
+	hw->switch_dev = switch_dev;
+}
+
 int
 i40e_dev_link_update(struct rte_eth_dev *dev,
 		     int wait_to_complete)
@@ -2803,6 +2820,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..9d77c85 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,8 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+void
+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
+struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v6 01/17] net/i40e: i40e support ipn3ke FPGA port bonding

[Zhang, Qi Z]

> -----Original Message-----
> From: Pei, Andy
> Sent: Thursday, September 19, 2019 5:03 PM
> To: dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>;
> Ye, Xiaolong <xiaolong.ye@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>;
> Lomartire, David <david.lomartire@intel.com>; Yigit, Ferruh
> <ferruh.yigit@intel.com>
> Subject: [PATCH v6 01/17] net/i40e: i40e support ipn3ke FPGA port bonding
> 
> In ipn3ke, each FPGA network side port bonding to an i40e pf, each i40e pf link
> status should get data from FPGA network, side port. This patch provide
> bonding relationship.
> 
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/net/i40e/base/i40e_type.h |  3 +++
>  drivers/net/i40e/i40e_ethdev.c    | 20 ++++++++++++++++++++
>  drivers/net/i40e/rte_pmd_i40e.h   |  4 ++++
>  3 files changed, 27 insertions(+)
> 
> diff --git a/drivers/net/i40e/base/i40e_type.h
> b/drivers/net/i40e/base/i40e_type.h
> index 112866b..06863d7 100644
> --- a/drivers/net/i40e/base/i40e_type.h
> +++ b/drivers/net/i40e/base/i40e_type.h
> @@ -660,6 +660,9 @@ struct i40e_hw {
>  	struct i40e_nvm_info nvm;
>  	struct i40e_fc_info fc;
> 
> +	/* switch device is used to get link status when i40e is in ipn3ke */
> +	struct rte_eth_dev *switch_dev;
> +
>  	/* pci info */
>  	u16 device_id;
>  	u16 vendor_id;
> diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
> index 4e40b7a..6c4b1e8 100644
> --- a/drivers/net/i40e/i40e_ethdev.c
> +++ b/drivers/net/i40e/i40e_ethdev.c
> @@ -1312,6 +1312,9 @@ static inline void i40e_config_automask(struct
> i40e_pf *pf)
>  	hw->adapter_stopped = 0;
>  	hw->adapter_closed = 0;
> 
> +	/* Init switch device pointer */
> +	hw->switch_dev = NULL;
> +
>  	/*
>  	 * Switch Tag value should not be identical to either the First Tag
>  	 * or Second Tag values. So set something other than common Ethertype
> @@ -2782,6 +2785,20 @@ void i40e_flex_payload_reg_set_default(struct
> i40e_hw *hw)
>  	}
>  }
> 
> +void
> +i40e_set_switch_dev(struct rte_eth_dev *i40e_dev, struct rte_eth_dev
> +*switch_dev) {
> +	struct i40e_hw *hw;
> +
> +	if (!i40e_dev)
> +		return;
> +
> +	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
> +
> +	hw->switch_dev = switch_dev;
> +}

Better to move the function implementation into rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
> +
>  int
>  i40e_dev_link_update(struct rte_eth_dev *dev,
>  		     int wait_to_complete)
> @@ -2803,6 +2820,9 @@ void i40e_flex_payload_reg_set_default(struct
> i40e_hw *hw)
>  	else
>  		update_link_aq(hw, &link, enable_lse, wait_to_complete);
> 
> +	if (hw->switch_dev)
> +		rte_eth_linkstatus_get(hw->switch_dev, &link);
> +
>  	ret = rte_eth_linkstatus_set(dev, &link);
>  	i40e_notify_all_vfs_link_status(dev);
> 
> diff --git a/drivers/net/i40e/rte_pmd_i40e.h
> b/drivers/net/i40e/rte_pmd_i40e.h index faac9e2..9d77c85 100644
> --- a/drivers/net/i40e/rte_pmd_i40e.h
> +++ b/drivers/net/i40e/rte_pmd_i40e.h
> @@ -1061,4 +1061,8 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t
> pctype,
>  	return 0;
>  }
> 
> +void
> +i40e_set_switch_dev(struct rte_eth_dev *i40e_dev, struct rte_eth_dev
> +*switch_dev);


1. Missing doxygen header here for the new API.
2. Also as an external API, we should use port_id but not rte_eth_dev as parameter.
3. you may also need to update the rte_pmd_i40e_version.map.

Regards
Qi

> +
>  #endif /* _PMD_I40E_H_ */
> --
> 1.8.3.1

Re: [dpdk-dev] [PATCH v6 01/17] net/i40e: i40e support ipn3ke FPGA port bonding

[Pei, Andy]

Hi Qi,

Will modify in next version.

-----Original Message-----
From: Zhang, Qi Z 
Sent: Friday, September 20, 2019 8:55 AM
To: Pei, Andy <andy.pei@intel.com>; dev@dpdk.org
Cc: Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>; Lomartire, David <david.lomartire@intel.com>; Yigit, Ferruh <ferruh.yigit@intel.com>
Subject: RE: [PATCH v6 01/17] net/i40e: i40e support ipn3ke FPGA port bonding



> -----Original Message-----
> From: Pei, Andy
> Sent: Thursday, September 19, 2019 5:03 PM
> To: dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei 
> <tianfei.zhang@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>; 
> Zhang, Qi Z <qi.z.zhang@intel.com>; Lomartire, David 
> <david.lomartire@intel.com>; Yigit, Ferruh <ferruh.yigit@intel.com>
> Subject: [PATCH v6 01/17] net/i40e: i40e support ipn3ke FPGA port 
> bonding
> 
> In ipn3ke, each FPGA network side port bonding to an i40e pf, each 
> i40e pf link status should get data from FPGA network, side port. This 
> patch provide bonding relationship.
> 
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/net/i40e/base/i40e_type.h |  3 +++
>  drivers/net/i40e/i40e_ethdev.c    | 20 ++++++++++++++++++++
>  drivers/net/i40e/rte_pmd_i40e.h   |  4 ++++
>  3 files changed, 27 insertions(+)
> 
> diff --git a/drivers/net/i40e/base/i40e_type.h
> b/drivers/net/i40e/base/i40e_type.h
> index 112866b..06863d7 100644
> --- a/drivers/net/i40e/base/i40e_type.h
> +++ b/drivers/net/i40e/base/i40e_type.h
> @@ -660,6 +660,9 @@ struct i40e_hw {
>  	struct i40e_nvm_info nvm;
>  	struct i40e_fc_info fc;
> 
> +	/* switch device is used to get link status when i40e is in ipn3ke */
> +	struct rte_eth_dev *switch_dev;
> +
>  	/* pci info */
>  	u16 device_id;
>  	u16 vendor_id;
> diff --git a/drivers/net/i40e/i40e_ethdev.c 
> b/drivers/net/i40e/i40e_ethdev.c index 4e40b7a..6c4b1e8 100644
> --- a/drivers/net/i40e/i40e_ethdev.c
> +++ b/drivers/net/i40e/i40e_ethdev.c
> @@ -1312,6 +1312,9 @@ static inline void i40e_config_automask(struct 
> i40e_pf *pf)
>  	hw->adapter_stopped = 0;
>  	hw->adapter_closed = 0;
> 
> +	/* Init switch device pointer */
> +	hw->switch_dev = NULL;
> +
>  	/*
>  	 * Switch Tag value should not be identical to either the First Tag
>  	 * or Second Tag values. So set something other than common 
> Ethertype @@ -2782,6 +2785,20 @@ void 
> i40e_flex_payload_reg_set_default(struct
> i40e_hw *hw)
>  	}
>  }
> 
> +void
> +i40e_set_switch_dev(struct rte_eth_dev *i40e_dev, struct rte_eth_dev
> +*switch_dev) {
> +	struct i40e_hw *hw;
> +
> +	if (!i40e_dev)
> +		return;
> +
> +	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
> +
> +	hw->switch_dev = switch_dev;
> +}

Better to move the function implementation into rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
> +
>  int
>  i40e_dev_link_update(struct rte_eth_dev *dev,
>  		     int wait_to_complete)
> @@ -2803,6 +2820,9 @@ void i40e_flex_payload_reg_set_default(struct
> i40e_hw *hw)
>  	else
>  		update_link_aq(hw, &link, enable_lse, wait_to_complete);
> 
> +	if (hw->switch_dev)
> +		rte_eth_linkstatus_get(hw->switch_dev, &link);
> +
>  	ret = rte_eth_linkstatus_set(dev, &link);
>  	i40e_notify_all_vfs_link_status(dev);
> 
> diff --git a/drivers/net/i40e/rte_pmd_i40e.h 
> b/drivers/net/i40e/rte_pmd_i40e.h index faac9e2..9d77c85 100644
> --- a/drivers/net/i40e/rte_pmd_i40e.h
> +++ b/drivers/net/i40e/rte_pmd_i40e.h
> @@ -1061,4 +1061,8 @@ int rte_pmd_i40e_inset_set(uint16_t port, 
> uint8_t pctype,
>  	return 0;
>  }
> 
> +void
> +i40e_set_switch_dev(struct rte_eth_dev *i40e_dev, struct rte_eth_dev 
> +*switch_dev);


1. Missing doxygen header here for the new API.
2. Also as an external API, we should use port_id but not rte_eth_dev as parameter.
3. you may also need to update the rte_pmd_i40e_version.map.

Regards
Qi

> +
>  #endif /* _PMD_I40E_H_ */
> --
> 1.8.3.1

Re: [dpdk-dev] [PATCH v6 01/17] net/i40e: i40e support ipn3ke FPGA port bonding

[Ye Xiaolong]

On 09/19, Andy Pei wrote:
>In ipn3ke, each FPGA network side port bonding to an i40e pf,
>each i40e pf link status should get data from FPGA network,
>side port. This patch provide bonding relationship.
>
>Signed-off-by: Rosen Xu <rosen.xu@intel.com>
>Signed-off-by: Andy Pei <andy.pei@intel.com>
>---
> drivers/net/i40e/base/i40e_type.h |  3 +++
> drivers/net/i40e/i40e_ethdev.c    | 20 ++++++++++++++++++++
> drivers/net/i40e/rte_pmd_i40e.h   |  4 ++++
> 3 files changed, 27 insertions(+)
>
>diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
>index 112866b..06863d7 100644
>--- a/drivers/net/i40e/base/i40e_type.h
>+++ b/drivers/net/i40e/base/i40e_type.h
>@@ -660,6 +660,9 @@ struct i40e_hw {
> 	struct i40e_nvm_info nvm;
> 	struct i40e_fc_info fc;
> 
>+	/* switch device is used to get link status when i40e is in ipn3ke */
>+	struct rte_eth_dev *switch_dev;
>+
> 	/* pci info */
> 	u16 device_id;
> 	u16 vendor_id;
>diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
>index 4e40b7a..6c4b1e8 100644
>--- a/drivers/net/i40e/i40e_ethdev.c
>+++ b/drivers/net/i40e/i40e_ethdev.c
>@@ -1312,6 +1312,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
> 	hw->adapter_stopped = 0;
> 	hw->adapter_closed = 0;
> 
>+	/* Init switch device pointer */
>+	hw->switch_dev = NULL;
>+
> 	/*
> 	 * Switch Tag value should not be identical to either the First Tag
> 	 * or Second Tag values. So set something other than common Ethertype
>@@ -2782,6 +2785,20 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
> 	}
> }
> 
>+void
>+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
>+struct rte_eth_dev *switch_dev)

Put a few tabs before parameter.

>+{
>+	struct i40e_hw *hw;
>+
>+	if (!i40e_dev)
>+		return;
>+
>+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
>+
>+	hw->switch_dev = switch_dev;
>+}
>+
> int
> i40e_dev_link_update(struct rte_eth_dev *dev,
> 		     int wait_to_complete)
>@@ -2803,6 +2820,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
> 	else
> 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
> 
>+	if (hw->switch_dev)
>+		rte_eth_linkstatus_get(hw->switch_dev, &link);
>+
> 	ret = rte_eth_linkstatus_set(dev, &link);
> 	i40e_notify_all_vfs_link_status(dev);
> 
>diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
>index faac9e2..9d77c85 100644
>--- a/drivers/net/i40e/rte_pmd_i40e.h
>+++ b/drivers/net/i40e/rte_pmd_i40e.h
>@@ -1061,4 +1061,8 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
> 	return 0;
> }
> 
>+void
>+i40e_set_switch_dev(struct rte_eth_dev *i40e_dev,
>+struct rte_eth_dev *switch_dev);

Ditto.

>+
> #endif /* _PMD_I40E_H_ */
>-- 
>1.8.3.1
>

[dpdk-dev] [PATCH v7 00/17] add PCIe AER disable and IRQ support for ipn3ke

[Andy Pei]

This patch set adds PCIe AER disable and IRQ support for ipn3ke.
Disable PCIe AER is very useful when FPGA reload. IRQ is used very
widely in interrupt process.

For ipn3ke is connect to CPU with PCIe switch, driver needs to scan
all PCIe devices of ipn3ke, it also can get all i40e of card, so
ipn3ke driver doesn't need to take some configuration of i40e.

v7 updates:
==========
- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
- function rte_pmd_i40e_set_switch_dev works as an external API,
  use port_id but not rte_eth_dev as parameter.
- add doxygen header here for the new API.
- update the rte_pmd_i40e_version.map.
- fix coding style issue.
- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
- for functions with a lot of similarity, extract out common function to reduce
  duplication.

v6 updates:
=========
- correct author information.
- correct typo in commit message and remove Gerrit Change-Id's before
  submitting upstream

v5 updates:
==========
- add lightweight fpga image support. in lightweight fpga image mode,
  ipn3ke representor will not be probed.

v4 updates:
===========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support

Andy Pei (2):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga: add lightweight fpga image support

Rosen Xu (3):
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei zhang (12):
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga/base: add secure support
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors
  raw/ifpga/base: add new API get board info

 config/common_base                            |   2 +-
 config/common_linux                           |   6 +
 drivers/net/i40e/base/i40e_type.h             |   3 +
 drivers/net/i40e/i40e_ethdev.c                |   6 +
 drivers/net/i40e/rte_pmd_i40e.c               |  21 +
 drivers/net/i40e/rte_pmd_i40e.h               |  17 +
 drivers/net/i40e/rte_pmd_i40e_version.map     |   8 +-
 drivers/net/ipn3ke/Makefile                   |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 291 ++-------
 drivers/net/ipn3ke/ipn3ke_representor.c       |   8 +-
 drivers/raw/ifpga/base/ifpga_api.c            |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h        |  75 ++-
 drivers/raw/ifpga/base/ifpga_feature_dev.c    |  59 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c            | 138 +++-
 drivers/raw/ifpga/base/ifpga_fme_error.c      |  74 ++-
 drivers/raw/ifpga/base/ifpga_hw.h             |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c           |  18 +
 drivers/raw/ifpga/base/ifpga_port_error.c     |  19 +
 drivers/raw/ifpga/base/opae_hw_api.c          | 135 ++++
 drivers/raw/ifpga/base/opae_hw_api.h          |  21 +
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c     | 568 ++++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h     | 146 ++++-
 drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
 drivers/raw/ifpga/base/opae_spi.h             |  23 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  40 +-
 drivers/raw/ifpga/ifpga_rawdev.c              | 874 +++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h              |  16 +
 mk/rte.app.mk                                 |   2 +-
 30 files changed, 2220 insertions(+), 387 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v7 01/17] net/i40e: i40e support ipn3ke FPGA port bonding

[Andy Pei]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h         |  3 +++
 drivers/net/i40e/i40e_ethdev.c            |  6 ++++++
 drivers/net/i40e/rte_pmd_i40e.c           | 21 +++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h           | 17 +++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e_version.map |  8 +++++++-
 5 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 4e40b7a..c88601c 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1312,6 +1312,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2803,6 +2806,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index 4c3c708..02d1572 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -3207,3 +3207,24 @@ int rte_pmd_i40e_flow_add_del_packet_template(
 	I40E_WRITE_FLUSH(hw);
 	return 0;
 }
+
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev)
+{
+	struct rte_eth_dev *i40e_dev;
+	struct i40e_hw *hw;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+	i40e_dev = &rte_eth_devices[port_id];
+	if (!is_i40e_supported(i40e_dev))
+		return -ENOTSUP;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+	if (!hw)
+		return -1;
+
+	hw->switch_dev = switch_dev;
+
+	return 0;
+}
\ No newline at end of file
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..51ef957 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,21 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+/**
+ * For ipn3ke, i40e works with FPGA.
+ * In this situation, i40e get link status from fpga,
+ * fpga works as switch_dev for i40e.
+ * This function set switch_dev for i40e.
+ *
+ * @param inset
+ *    Input set value.
+ * @param field_idx
+ *    Field index for input set.
+ * @return
+ *   - (less than 0) if failed.
+ *   - (0) if success.
+ */
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
index cccd576..af9b304 100644
--- a/drivers/net/i40e/rte_pmd_i40e_version.map
+++ b/drivers/net/i40e/rte_pmd_i40e_version.map
@@ -64,4 +64,10 @@ DPDK_18.02 {
 
 	rte_pmd_i40e_inset_get;
 	rte_pmd_i40e_inset_set;
-} DPDK_17.11;
\ No newline at end of file
+} DPDK_17.11;
+
+DPDK_19.11 {
+	global:
+
+	rte_pmd_i40e_set_switch_dev;
+} DPDK_18.02;
\ No newline at end of file
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 00/18] add PCIe AER disable and IRQ support for ipn3ke

[Andy Pei]

This patch set adds PCIe AER disable and IRQ support for ipn3ke.
Disable PCIe AER is very useful when FPGA reload. IRQ is used very
widely in interrupt process.

For ipn3ke is connect to CPU with PCIe switch, driver needs to scan
all PCIe devices of ipn3ke, it also can get all i40e of card, so
ipn3ke driver doesn't need to take some configuration of i40e.


v8 updates:
=========
- add multiple cards support.

v7 updates:
==========
- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
- function rte_pmd_i40e_set_switch_dev works as an external API,
  use port_id but not rte_eth_dev as parameter.
- add doxygen header here for the new API.
- update the rte_pmd_i40e_version.map.
- fix coding style issue.
- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
- for functions with a lot of similarity, extract out common function to reduce
  duplication.

v6 updates:
=========
- correct author information.
- correct typo in commit message and remove Gerrit Change-Id's before
  submitting upstream

v5 updates:
==========
- add lightweight fpga image support. in lightweight fpga image mode,
  ipn3ke representor will not be probed.

v4 updates:
===========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support

Andy Pei (2):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga: add lightweight fpga image support

Rosen Xu (3):
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei zhang (13):
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga/base: add secure support
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors
  raw/ifpga/base: add new API get board info
  raw/ifpga/base: add multiple cards support

 config/common_base                            |   4 +-
 config/common_linux                           |   6 +
 drivers/net/i40e/base/i40e_type.h             |   3 +
 drivers/net/i40e/i40e_ethdev.c                |   6 +
 drivers/net/i40e/rte_pmd_i40e.c               |  21 +
 drivers/net/i40e/rte_pmd_i40e.h               |  18 +
 drivers/net/i40e/rte_pmd_i40e_version.map     |   8 +-
 drivers/net/ipn3ke/Makefile                   |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 289 ++-------
 drivers/net/ipn3ke/ipn3ke_representor.c       |   8 +-
 drivers/raw/ifpga/base/ifpga_api.c            |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h        |  75 ++-
 drivers/raw/ifpga/base/ifpga_feature_dev.c    |  59 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c            | 166 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c      |  74 ++-
 drivers/raw/ifpga/base/ifpga_hw.h             |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c           |  18 +
 drivers/raw/ifpga/base/ifpga_port_error.c     |  19 +
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          | 137 ++++
 drivers/raw/ifpga/base/opae_hw_api.h          |  26 +
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c     | 598 +++++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h     | 157 ++++-
 drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
 drivers/raw/ifpga/base/opae_spi.c             |   1 +
 drivers/raw/ifpga/base/opae_spi.h             |  25 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  40 +-
 drivers/raw/ifpga/ifpga_rawdev.c              | 860 +++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h              |  16 +
 mk/rte.app.mk                                 |   2 +-
 32 files changed, 2277 insertions(+), 399 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v8 01/18] net/i40e: i40e support ipn3ke FPGA port bonding

[Andy Pei]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h         |  3 +++
 drivers/net/i40e/i40e_ethdev.c            |  6 ++++++
 drivers/net/i40e/rte_pmd_i40e.c           | 21 +++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h           | 18 ++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e_version.map |  8 +++++++-
 5 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 2ca14da..bae1ca2 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1390,6 +1390,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2901,6 +2904,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index 4c3c708..fdcb1a4 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -3207,3 +3207,24 @@ int rte_pmd_i40e_flow_add_del_packet_template(
 	I40E_WRITE_FLUSH(hw);
 	return 0;
 }
+
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev)
+{
+	struct rte_eth_dev *i40e_dev;
+	struct i40e_hw *hw;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+	i40e_dev = &rte_eth_devices[port_id];
+	if (!is_i40e_supported(i40e_dev))
+		return -ENOTSUP;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+	if (!hw)
+		return -1;
+
+	hw->switch_dev = switch_dev;
+
+	return 0;
+}
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..355b8be 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+/**
+ * For ipn3ke, i40e works with FPGA.
+ * In this situation, i40e get link status from fpga,
+ * fpga works as switch_dev for i40e.
+ * This function set switch_dev for i40e.
+ *
+ * @param inset
+ *    Input set value.
+ * @param field_idx
+ *    Field index for input set.
+ * @return
+ *   - (less than 0) if failed.
+ *   - (0) if success.
+ */
+__rte_experimental
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
index cccd576..79641f2 100644
--- a/drivers/net/i40e/rte_pmd_i40e_version.map
+++ b/drivers/net/i40e/rte_pmd_i40e_version.map
@@ -64,4 +64,10 @@ DPDK_18.02 {
 
 	rte_pmd_i40e_inset_get;
 	rte_pmd_i40e_inset_set;
-} DPDK_17.11;
\ No newline at end of file
+} DPDK_17.11;
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_i40e_set_switch_dev;
+} DPDK_18.02;
\ No newline at end of file
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 00/18] add PCIe AER disable and IRQ support for ipn3ke

[Andy Pei]

This patch set adds PCIe AER disable and IRQ support for ipn3ke.
Disable PCIe AER is very useful when FPGA reload. IRQ is used very
widely in interrupt process.

For ipn3ke is connect to CPU with PCIe switch, driver needs to scan
all PCIe devices of ipn3ke, it also can get all i40e of card, so
ipn3ke driver doesn't need to take some configuration of i40e.


v9 updates:
=========
- Add mutex lock on do_transaction() function for SPI driver to avoid race 
  condition.

v8 updates:
=========
- add multiple cards support.

v7 updates:
==========
- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
- function rte_pmd_i40e_set_switch_dev works as an external API,
  use port_id but not rte_eth_dev as parameter.
- add doxygen header here for the new API.
- update the rte_pmd_i40e_version.map.
- fix coding style issue.
- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
- for functions with a lot of similarity, extract out common function to reduce
  duplication.

v6 updates:
=========
- correct author information.
- correct typo in commit message and remove Gerrit Change-Id's before
  submitting upstream

v5 updates:
==========
- add lightweight fpga image support. in lightweight fpga image mode,
  ipn3ke representor will not be probed.

v4 updates:
===========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support

Andy Pei (2):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga: add lightweight fpga image support

Rosen Xu (3):
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei zhang (13):
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga/base: add secure support
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors
  raw/ifpga/base: add new API get board info
  raw/ifpga/base: add multiple cards support

 config/common_base                            |   4 +-
 config/common_linux                           |   6 +
 drivers/net/i40e/base/i40e_type.h             |   3 +
 drivers/net/i40e/i40e_ethdev.c                |   6 +
 drivers/net/i40e/rte_pmd_i40e.c               |  21 +
 drivers/net/i40e/rte_pmd_i40e.h               |  18 +
 drivers/net/i40e/rte_pmd_i40e_version.map     |   8 +-
 drivers/net/ipn3ke/Makefile                   |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 289 ++-------
 drivers/net/ipn3ke/ipn3ke_representor.c       |   8 +-
 drivers/raw/ifpga/base/ifpga_api.c            |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h        |  75 ++-
 drivers/raw/ifpga/base/ifpga_feature_dev.c    |  59 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c            | 166 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c      |  74 ++-
 drivers/raw/ifpga/base/ifpga_hw.h             |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c           |  18 +
 drivers/raw/ifpga/base/ifpga_port_error.c     |  19 +
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          | 137 ++++
 drivers/raw/ifpga/base/opae_hw_api.h          |  26 +
 drivers/raw/ifpga/base/opae_i2c.c             |  44 +-
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c     | 598 +++++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h     | 157 ++++-
 drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
 drivers/raw/ifpga/base/opae_spi.c             |   5 -
 drivers/raw/ifpga/base/opae_spi.h             |  26 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  84 ++-
 drivers/raw/ifpga/ifpga_rawdev.c              | 860 +++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h              |  16 +
 mk/rte.app.mk                                 |   2 +-
 34 files changed, 2348 insertions(+), 424 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v9 01/18] net/i40e: i40e support ipn3ke FPGA port bonding

[Andy Pei]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h         |  3 +++
 drivers/net/i40e/i40e_ethdev.c            |  6 ++++++
 drivers/net/i40e/rte_pmd_i40e.c           | 21 +++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h           | 18 ++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e_version.map |  8 +++++++-
 5 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 2ca14da..bae1ca2 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1390,6 +1390,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2901,6 +2904,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index 4c3c708..fdcb1a4 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -3207,3 +3207,24 @@ int rte_pmd_i40e_flow_add_del_packet_template(
 	I40E_WRITE_FLUSH(hw);
 	return 0;
 }
+
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev)
+{
+	struct rte_eth_dev *i40e_dev;
+	struct i40e_hw *hw;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+	i40e_dev = &rte_eth_devices[port_id];
+	if (!is_i40e_supported(i40e_dev))
+		return -ENOTSUP;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+	if (!hw)
+		return -1;
+
+	hw->switch_dev = switch_dev;
+
+	return 0;
+}
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..355b8be 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+/**
+ * For ipn3ke, i40e works with FPGA.
+ * In this situation, i40e get link status from fpga,
+ * fpga works as switch_dev for i40e.
+ * This function set switch_dev for i40e.
+ *
+ * @param inset
+ *    Input set value.
+ * @param field_idx
+ *    Field index for input set.
+ * @return
+ *   - (less than 0) if failed.
+ *   - (0) if success.
+ */
+__rte_experimental
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
index cccd576..79641f2 100644
--- a/drivers/net/i40e/rte_pmd_i40e_version.map
+++ b/drivers/net/i40e/rte_pmd_i40e_version.map
@@ -64,4 +64,10 @@ DPDK_18.02 {
 
 	rte_pmd_i40e_inset_get;
 	rte_pmd_i40e_inset_set;
-} DPDK_17.11;
\ No newline at end of file
+} DPDK_17.11;
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_i40e_set_switch_dev;
+} DPDK_18.02;
\ No newline at end of file
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Andy Pei]

This patch set adds PCIe AER disable and FPGA interrupt support for 
ipn3ke. It also provides a small rework for port bonding between FPGA 
line side port and I40e PF port.

What is the PCI Express AER(Advanced Error Reporting)?
Advanced Error Reporting capability is implemented with a PCI Express
advanced error reporting extended capability structure providing more
robust error reporting. It's also one of PCI Express error reporting
paradigms. AER is supported by most of PCIe devices.

In PAC N3000 card, some uncertainty errors will cause FPGA reload,
such as temperature is higher than threshold. From Software point of
view, FPGA reload means FPGA unplug and plug. For avoiding system
crash we need to clear AER register before these errors occur.

Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga
rawdev driver, we implement a FME interrupt function to notify errors
reported by FME. Besides this, OPAE share code also provide a common
AFU interrupt API for users to register their own interrupt functions.


Andy Pei (2):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga: add lightweight fpga image support

Rosen Xu (3):
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei zhang (14):
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga/base: add secure support
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors
  raw/ifpga/base: add new API get board info
  raw/ifpga/base: add multiple cards support
  raw/ifpga: introducing new irq API

 config/common_base                             |   4 +-
 config/common_linux                            |   6 +
 drivers/meson.build                            |   7 +-
 drivers/net/i40e/base/i40e_type.h              |   3 +
 drivers/net/i40e/i40e_ethdev.c                 |   6 +
 drivers/net/i40e/rte_pmd_i40e.c                |  21 +
 drivers/net/i40e/rte_pmd_i40e.h                |  18 +
 drivers/net/i40e/rte_pmd_i40e_version.map      |   8 +-
 drivers/net/ipn3ke/Makefile                    |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c             | 289 +-------
 drivers/net/ipn3ke/ipn3ke_representor.c        |   8 +-
 drivers/net/ipn3ke/meson.build                 |   2 +-
 drivers/raw/ifpga/base/ifpga_api.c             |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h         |  75 +-
 drivers/raw/ifpga/base/ifpga_feature_dev.c     |  59 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h     |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c             | 166 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c       |  74 +-
 drivers/raw/ifpga/base/ifpga_hw.h              |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c            |  18 +
 drivers/raw/ifpga/base/ifpga_port_error.c      |  19 +
 drivers/raw/ifpga/base/opae_debug.c            |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c           | 137 ++++
 drivers/raw/ifpga/base/opae_hw_api.h           |  26 +
 drivers/raw/ifpga/base/opae_i2c.c              |  44 +-
 drivers/raw/ifpga/base/opae_i2c.h              |   3 +-
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h     |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c      | 599 +++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h      | 157 ++++-
 drivers/raw/ifpga/base/opae_osdep.h            |   7 +-
 drivers/raw/ifpga/base/opae_spi.c              |   5 -
 drivers/raw/ifpga/base/opae_spi.h              |  26 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c  |  84 ++-
 drivers/raw/ifpga/ifpga_rawdev.c               | 903 ++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h               |  30 +
 drivers/raw/ifpga/meson.build                  |   6 +
 drivers/raw/ifpga/rte_rawdev_ifpga_version.map |   3 +
 mk/rte.app.mk                                  |   2 +-
 38 files changed, 2421 insertions(+), 427 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v10 01/19] net/i40e: i40e support ipn3ke FPGA port bonding

[Andy Pei]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h         |  3 +++
 drivers/net/i40e/i40e_ethdev.c            |  6 ++++++
 drivers/net/i40e/rte_pmd_i40e.c           | 21 +++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h           | 18 ++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e_version.map |  8 +++++++-
 5 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 2ca14da..bae1ca2 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1390,6 +1390,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2901,6 +2904,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index 4c3c708..fdcb1a4 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -3207,3 +3207,24 @@ int rte_pmd_i40e_flow_add_del_packet_template(
 	I40E_WRITE_FLUSH(hw);
 	return 0;
 }
+
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev)
+{
+	struct rte_eth_dev *i40e_dev;
+	struct i40e_hw *hw;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+	i40e_dev = &rte_eth_devices[port_id];
+	if (!is_i40e_supported(i40e_dev))
+		return -ENOTSUP;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+	if (!hw)
+		return -1;
+
+	hw->switch_dev = switch_dev;
+
+	return 0;
+}
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..355b8be 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+/**
+ * For ipn3ke, i40e works with FPGA.
+ * In this situation, i40e get link status from fpga,
+ * fpga works as switch_dev for i40e.
+ * This function set switch_dev for i40e.
+ *
+ * @param inset
+ *    Input set value.
+ * @param field_idx
+ *    Field index for input set.
+ * @return
+ *   - (less than 0) if failed.
+ *   - (0) if success.
+ */
+__rte_experimental
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
index cccd576..79641f2 100644
--- a/drivers/net/i40e/rte_pmd_i40e_version.map
+++ b/drivers/net/i40e/rte_pmd_i40e_version.map
@@ -64,4 +64,10 @@ DPDK_18.02 {
 
 	rte_pmd_i40e_inset_get;
 	rte_pmd_i40e_inset_set;
-} DPDK_17.11;
\ No newline at end of file
+} DPDK_17.11;
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_i40e_set_switch_dev;
+} DPDK_18.02;
\ No newline at end of file
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Andy Pei]

This patch set adds PCIe AER disable and FPGA interrupt support for
ipn3ke. It also provides a small rework for port bonding between FPGA
line side port and I40e PF port.

What is the PCI Express AER(Advanced Error Reporting)?
Advanced Error Reporting capability is implemented with a PCI Express
advanced error reporting extended capability structure providing more
robust error reporting. It's also one of PCI Express error reporting
paradigms. AER is supported by most of PCIe devices.

In PAC N3000 card, some uncertainty errors will cause FPGA reload,
such as temperature is higher than threshold. From Software point of
view, FPGA reload means FPGA unplug and plug. For avoiding system
crash we need to clear AER register before these errors occur.

Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga
rawdev driver, we implement a FME interrupt function to notify errors
reported by FME. Besides this, OPAE share code also provide a common
AFU interrupt API for users to register their own interrupt functions.

v11 updates:
=========
- move symbol ifpga_rawdev_ge to the EXPERIMENTAL section of 
  drivers/raw/ifpga/rte_rawdev_ifpga_version.map

v10 updates:
=========
- introducing new irq API
- fix meson build issue

v9 updates:
=========
- Add mutex lock on do_transaction() function for SPI driver to avoid race
  condition.

v8 updates:
=========
- add multiple cards support.

v7 updates:
==========
- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
- function rte_pmd_i40e_set_switch_dev works as an external API,
  use port_id but not rte_eth_dev as parameter.
- add doxygen header here for the new API.
- update the rte_pmd_i40e_version.map.
- fix coding style issue.
- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
- for functions with a lot of similarity, extract out common function to reduce
  duplication.

v6 updates:
=========
- correct author information.
- correct typo in commit message and remove Gerrit Change-Id's before
  submitting upstream

v5 updates:
==========
- add lightweight fpga image support. in lightweight fpga image mode,
  ipn3ke representor will not be probed.

v4 updates:
===========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support


Andy Pei (2):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga: add lightweight fpga image support

Rosen Xu (3):
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei zhang (14):
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga/base: add secure support
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors
  raw/ifpga/base: add new API get board info
  raw/ifpga/base: add multiple cards support
  raw/ifpga: introducing new irq API

 config/common_base                             |   4 +-
 config/common_linux                            |   6 +
 drivers/meson.build                            |   7 +-
 drivers/net/i40e/base/i40e_type.h              |   3 +
 drivers/net/i40e/i40e_ethdev.c                 |   6 +
 drivers/net/i40e/rte_pmd_i40e.c                |  21 +
 drivers/net/i40e/rte_pmd_i40e.h                |  18 +
 drivers/net/i40e/rte_pmd_i40e_version.map      |   8 +-
 drivers/net/ipn3ke/Makefile                    |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c             | 289 +-------
 drivers/net/ipn3ke/ipn3ke_representor.c        |   8 +-
 drivers/net/ipn3ke/meson.build                 |   2 +-
 drivers/raw/ifpga/base/ifpga_api.c             |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h         |  75 +-
 drivers/raw/ifpga/base/ifpga_feature_dev.c     |  59 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h     |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c             | 166 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c       |  74 +-
 drivers/raw/ifpga/base/ifpga_hw.h              |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c            |  18 +
 drivers/raw/ifpga/base/ifpga_port_error.c      |  19 +
 drivers/raw/ifpga/base/opae_debug.c            |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c           | 137 ++++
 drivers/raw/ifpga/base/opae_hw_api.h           |  26 +
 drivers/raw/ifpga/base/opae_i2c.c              |  44 +-
 drivers/raw/ifpga/base/opae_i2c.h              |   3 +-
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h     |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c      | 599 +++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h      | 157 ++++-
 drivers/raw/ifpga/base/opae_osdep.h            |   7 +-
 drivers/raw/ifpga/base/opae_spi.c              |   5 -
 drivers/raw/ifpga/base/opae_spi.h              |  26 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c  |  84 ++-
 drivers/raw/ifpga/ifpga_rawdev.c               | 903 ++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h               |  30 +
 drivers/raw/ifpga/meson.build                  |   6 +
 drivers/raw/ifpga/rte_rawdev_ifpga_version.map |   6 +
 mk/rte.app.mk                                  |   2 +-
 38 files changed, 2424 insertions(+), 427 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v11 01/19] net/i40e: i40e support ipn3ke FPGA port bonding

[Andy Pei]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h         |  3 +++
 drivers/net/i40e/i40e_ethdev.c            |  6 ++++++
 drivers/net/i40e/rte_pmd_i40e.c           | 21 +++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h           | 18 ++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e_version.map |  8 +++++++-
 5 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 2ca14da..bae1ca2 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1390,6 +1390,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2901,6 +2904,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index 4c3c708..fdcb1a4 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -3207,3 +3207,24 @@ int rte_pmd_i40e_flow_add_del_packet_template(
 	I40E_WRITE_FLUSH(hw);
 	return 0;
 }
+
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev)
+{
+	struct rte_eth_dev *i40e_dev;
+	struct i40e_hw *hw;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+	i40e_dev = &rte_eth_devices[port_id];
+	if (!is_i40e_supported(i40e_dev))
+		return -ENOTSUP;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+	if (!hw)
+		return -1;
+
+	hw->switch_dev = switch_dev;
+
+	return 0;
+}
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..355b8be 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+/**
+ * For ipn3ke, i40e works with FPGA.
+ * In this situation, i40e get link status from fpga,
+ * fpga works as switch_dev for i40e.
+ * This function set switch_dev for i40e.
+ *
+ * @param inset
+ *    Input set value.
+ * @param field_idx
+ *    Field index for input set.
+ * @return
+ *   - (less than 0) if failed.
+ *   - (0) if success.
+ */
+__rte_experimental
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
index cccd576..79641f2 100644
--- a/drivers/net/i40e/rte_pmd_i40e_version.map
+++ b/drivers/net/i40e/rte_pmd_i40e_version.map
@@ -64,4 +64,10 @@ DPDK_18.02 {
 
 	rte_pmd_i40e_inset_get;
 	rte_pmd_i40e_inset_set;
-} DPDK_17.11;
\ No newline at end of file
+} DPDK_17.11;
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_i40e_set_switch_dev;
+} DPDK_18.02;
\ No newline at end of file
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Andy Pei]

This patch set adds PCIe AER disable and FPGA interrupt support for
ipn3ke. It also provides a small rework for port bonding between FPGA
line side port and I40e PF port.

What is the PCI Express AER(Advanced Error Reporting)?
Advanced Error Reporting capability is implemented with a PCI Express
advanced error reporting extended capability structure providing more
robust error reporting. It's also one of PCI Express error reporting
paradigms. AER is supported by most of PCIe devices.

In PAC N3000 card, some uncertainty errors will cause FPGA reload,
such as temperature is higher than threshold. From Software point of
view, FPGA reload means FPGA unplug and plug. For avoiding system
crash we need to clear AER register before these errors occur.

Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga
rawdev driver, we implement a FME interrupt function to notify errors
reported by FME. Besides this, OPAE share code also provide a common
AFU interrupt API for users to register their own interrupt functions.

v11 updates:
=========
- fix meson build issue.
 
v11 updates:
=========
- move symbol ifpga_rawdev_ge to the EXPERIMENTAL section of 
  drivers/raw/ifpga/rte_rawdev_ifpga_version.map

v10 updates:
=========
- introducing new irq API
- fix meson build issue

v9 updates:
=========
- Add mutex lock on do_transaction() function for SPI driver to avoid race
  condition.

v8 updates:
=========
- add multiple cards support.

v7 updates:
==========
- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
- function rte_pmd_i40e_set_switch_dev works as an external API,
  use port_id but not rte_eth_dev as parameter.
- add doxygen header here for the new API.
- update the rte_pmd_i40e_version.map.
- fix coding style issue.
- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
- for functions with a lot of similarity, extract out common function to reduce
  duplication.

v6 updates:
=========
- correct author information.
- correct typo in commit message and remove Gerrit Change-Id's before
  submitting upstream

v5 updates:
==========
- add lightweight fpga image support. in lightweight fpga image mode,
  ipn3ke representor will not be probed.

v4 updates:
===========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support

Andy Pei (2):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga: add lightweight fpga image support

Rosen Xu (3):
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei zhang (14):
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga/base: add secure support
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors
  raw/ifpga/base: add new API get board info
  raw/ifpga/base: add multiple cards support
  raw/ifpga: introducing new irq API

 config/common_base                            |   4 +-
 config/common_linux                           |   6 +
 drivers/net/i40e/base/i40e_type.h             |   3 +
 drivers/net/i40e/i40e_ethdev.c                |   6 +
 drivers/net/i40e/rte_pmd_i40e.c               |  21 +
 drivers/net/i40e/rte_pmd_i40e.h               |  18 +
 drivers/net/i40e/rte_pmd_i40e_version.map     |   8 +-
 drivers/net/ipn3ke/Makefile                   |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 289 +--------
 drivers/net/ipn3ke/ipn3ke_representor.c       |   8 +-
 drivers/net/ipn3ke/meson.build                |  24 +-
 drivers/raw/ifpga/base/ifpga_api.c            |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h        |  75 ++-
 drivers/raw/ifpga/base/ifpga_feature_dev.c    |  59 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c            | 166 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c      |  74 ++-
 drivers/raw/ifpga/base/ifpga_hw.h             |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c           |  18 +
 drivers/raw/ifpga/base/ifpga_port_error.c     |  19 +
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          | 137 ++++
 drivers/raw/ifpga/base/opae_hw_api.h          |  26 +
 drivers/raw/ifpga/base/opae_i2c.c             |  44 +-
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c     | 599 ++++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h     | 157 ++++-
 drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
 drivers/raw/ifpga/base/opae_spi.c             |   5 -
 drivers/raw/ifpga/base/opae_spi.h             |  26 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  84 ++-
 drivers/raw/ifpga/ifpga_rawdev.c              | 903 +++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h              |  30 +
 drivers/raw/ifpga/meson.build                 |  25 +-
 mk/rte.app.mk                                 |   2 +-
 36 files changed, 2441 insertions(+), 438 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v12 01/19] net/i40e: i40e support ipn3ke FPGA port bonding

[Andy Pei]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h         |  3 +++
 drivers/net/i40e/i40e_ethdev.c            |  6 ++++++
 drivers/net/i40e/rte_pmd_i40e.c           | 21 +++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h           | 18 ++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e_version.map |  8 +++++++-
 5 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 2ca14da..bae1ca2 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1390,6 +1390,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2901,6 +2904,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index 4c3c708..fdcb1a4 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -3207,3 +3207,24 @@ int rte_pmd_i40e_flow_add_del_packet_template(
 	I40E_WRITE_FLUSH(hw);
 	return 0;
 }
+
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev)
+{
+	struct rte_eth_dev *i40e_dev;
+	struct i40e_hw *hw;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+	i40e_dev = &rte_eth_devices[port_id];
+	if (!is_i40e_supported(i40e_dev))
+		return -ENOTSUP;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+	if (!hw)
+		return -1;
+
+	hw->switch_dev = switch_dev;
+
+	return 0;
+}
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..355b8be 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+/**
+ * For ipn3ke, i40e works with FPGA.
+ * In this situation, i40e get link status from fpga,
+ * fpga works as switch_dev for i40e.
+ * This function set switch_dev for i40e.
+ *
+ * @param inset
+ *    Input set value.
+ * @param field_idx
+ *    Field index for input set.
+ * @return
+ *   - (less than 0) if failed.
+ *   - (0) if success.
+ */
+__rte_experimental
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
index cccd576..79641f2 100644
--- a/drivers/net/i40e/rte_pmd_i40e_version.map
+++ b/drivers/net/i40e/rte_pmd_i40e_version.map
@@ -64,4 +64,10 @@ DPDK_18.02 {
 
 	rte_pmd_i40e_inset_get;
 	rte_pmd_i40e_inset_set;
-} DPDK_17.11;
\ No newline at end of file
+} DPDK_17.11;
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_i40e_set_switch_dev;
+} DPDK_18.02;
\ No newline at end of file
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v12 01/19] net/i40e: i40e support ipn3ke FPGA port bonding

[Ye Xiaolong]

On 10/23, Andy Pei wrote:
>+EXPERIMENTAL {
>+	global:
>+
>+	rte_pmd_i40e_set_switch_dev;
>+} DPDK_18.02;

No need the versioning for the EXPERIMENTAL section.

>\ No newline at end of file

This is annoying, you can add "set nofixeol" in you vimrc to avoid this.

Thanks,
Xiaolong
>-- 
>1.8.3.1
>

[dpdk-dev] [PATCH v13 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Andy Pei]

This patch set adds PCIe AER disable and FPGA interrupt support for
ipn3ke. It also provides a small rework for port bonding between FPGA
line side port and I40e PF port.

What is the PCI Express AER(Advanced Error Reporting)?
Advanced Error Reporting capability is implemented with a PCI Express
advanced error reporting extended capability structure providing more
robust error reporting. It's also one of PCI Express error reporting
paradigms. AER is supported by most of PCIe devices.

In PAC N3000 card, some uncertainty errors will cause FPGA reload,
such as temperature is higher than threshold. From Software point of
view, FPGA reload means FPGA unplug and plug. For avoiding system
crash we need to clear AER register before these errors occur.

Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga
rawdev driver, we implement a FME interrupt function to notify errors
reported by FME. Besides this, OPAE share code also provide a common
AFU interrupt API for users to register their own interrupt functions.

v13 updates:
=========
- fix meson build issue.

v12 updates:
=========
- fix meson build issue.
 
v11 updates:
=========
- move symbol ifpga_rawdev_ge to the EXPERIMENTAL section of 
  drivers/raw/ifpga/rte_rawdev_ifpga_version.map

v10 updates:
=========
- introducing new irq API
- fix meson build issue

v9 updates:
=========
- Add mutex lock on do_transaction() function for SPI driver to avoid race
  condition.

v8 updates:
=========
- add multiple cards support.

v7 updates:
==========
- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
- function rte_pmd_i40e_set_switch_dev works as an external API,
  use port_id but not rte_eth_dev as parameter.
- add doxygen header here for the new API.
- update the rte_pmd_i40e_version.map.
- fix coding style issue.
- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
- for functions with a lot of similarity, extract out common function to reduce
  duplication.

v6 updates:
=========
- correct author information.
- correct typo in commit message and remove Gerrit Change-Id's before
  submitting upstream

v5 updates:
==========
- add lightweight fpga image support. in lightweight fpga image mode,
  ipn3ke representor will not be probed.

v4 updates:
===========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support

Andy Pei (2):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga: add lightweight fpga image support

Rosen Xu (3):
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei zhang (14):
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga/base: add secure support
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors
  raw/ifpga/base: add new API get board info
  raw/ifpga/base: add multiple cards support
  raw/ifpga: introducing new irq API

 config/common_base                             |   4 +-
 config/common_linux                            |   6 +
 drivers/meson.build                            |   6 +-
 drivers/net/i40e/base/i40e_type.h              |   3 +
 drivers/net/i40e/i40e_ethdev.c                 |   6 +
 drivers/net/i40e/rte_pmd_i40e.c                |  21 +
 drivers/net/i40e/rte_pmd_i40e.h                |  18 +
 drivers/net/i40e/rte_pmd_i40e_version.map      |   8 +-
 drivers/net/ipn3ke/Makefile                    |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c             | 289 +-------
 drivers/net/ipn3ke/ipn3ke_representor.c        |   8 +-
 drivers/net/ipn3ke/meson.build                 |  24 +-
 drivers/raw/ifpga/base/ifpga_api.c             |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h         |  75 +-
 drivers/raw/ifpga/base/ifpga_feature_dev.c     |  59 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h     |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c             | 166 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c       |  74 +-
 drivers/raw/ifpga/base/ifpga_hw.h              |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c            |  18 +
 drivers/raw/ifpga/base/ifpga_port_error.c      |  19 +
 drivers/raw/ifpga/base/meson.build             |   2 +-
 drivers/raw/ifpga/base/opae_debug.c            |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c           | 137 ++++
 drivers/raw/ifpga/base/opae_hw_api.h           |  26 +
 drivers/raw/ifpga/base/opae_i2c.c              |  44 +-
 drivers/raw/ifpga/base/opae_i2c.h              |   3 +-
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h     |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c      | 599 +++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h      | 157 ++++-
 drivers/raw/ifpga/base/opae_osdep.h            |   7 +-
 drivers/raw/ifpga/base/opae_spi.c              |   5 -
 drivers/raw/ifpga/base/opae_spi.h              |  26 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c  |  84 ++-
 drivers/raw/ifpga/ifpga_rawdev.c               | 903 ++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h               |  30 +
 drivers/raw/ifpga/meson.build                  |  25 +-
 drivers/raw/ifpga/rte_rawdev_ifpga_version.map |   3 +
 mk/rte.app.mk                                  |   2 +-
 39 files changed, 2448 insertions(+), 442 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v13 01/19] net/i40e: i40e support ipn3ke FPGA port bonding

[Andy Pei]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h         |  3 +++
 drivers/net/i40e/i40e_ethdev.c            |  6 ++++++
 drivers/net/i40e/rte_pmd_i40e.c           | 21 +++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h           | 18 ++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e_version.map |  8 +++++++-
 5 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 2ca14da..bae1ca2 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1390,6 +1390,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2901,6 +2904,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index 4c3c708..fdcb1a4 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -3207,3 +3207,24 @@ int rte_pmd_i40e_flow_add_del_packet_template(
 	I40E_WRITE_FLUSH(hw);
 	return 0;
 }
+
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev)
+{
+	struct rte_eth_dev *i40e_dev;
+	struct i40e_hw *hw;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+	i40e_dev = &rte_eth_devices[port_id];
+	if (!is_i40e_supported(i40e_dev))
+		return -ENOTSUP;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+	if (!hw)
+		return -1;
+
+	hw->switch_dev = switch_dev;
+
+	return 0;
+}
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..355b8be 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+/**
+ * For ipn3ke, i40e works with FPGA.
+ * In this situation, i40e get link status from fpga,
+ * fpga works as switch_dev for i40e.
+ * This function set switch_dev for i40e.
+ *
+ * @param inset
+ *    Input set value.
+ * @param field_idx
+ *    Field index for input set.
+ * @return
+ *   - (less than 0) if failed.
+ *   - (0) if success.
+ */
+__rte_experimental
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
index cccd576..79641f2 100644
--- a/drivers/net/i40e/rte_pmd_i40e_version.map
+++ b/drivers/net/i40e/rte_pmd_i40e_version.map
@@ -64,4 +64,10 @@ DPDK_18.02 {
 
 	rte_pmd_i40e_inset_get;
 	rte_pmd_i40e_inset_set;
-} DPDK_17.11;
\ No newline at end of file
+} DPDK_17.11;
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_i40e_set_switch_dev;
+} DPDK_18.02;
\ No newline at end of file
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Andy Pei]

This patch set adds PCIe AER disable and FPGA interrupt support for
ipn3ke. It also provides a small rework for port bonding between FPGA
line side port and I40e PF port.

What is the PCI Express AER(Advanced Error Reporting)?
Advanced Error Reporting capability is implemented with a PCI Express
advanced error reporting extended capability structure providing more
robust error reporting. It's also one of PCI Express error reporting
paradigms. AER is supported by most of PCIe devices.

In PAC N3000 card, some uncertainty errors will cause FPGA reload,
such as temperature is higher than threshold. From Software point of
view, FPGA reload means FPGA unplug and plug. For avoiding system
crash we need to clear AER register before these errors occur.

Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga
rawdev driver, we implement a FME interrupt function to notify errors
reported by FME. Besides this, OPAE share code also provide a common
AFU interrupt API for users to register their own interrupt functions.

v14 updates:
=========
- fix coding style and remove unnecessary comments.

v13 updates:
=========
- fix meson build issue.

v12 updates:
=========
- fix meson build issue.
 
v11 updates:
=========
- move symbol ifpga_rawdev_ge to the EXPERIMENTAL section of 
  drivers/raw/ifpga/rte_rawdev_ifpga_version.map

v10 updates:
=========
- introducing new irq API
- fix meson build issue

v9 updates:
=========
- Add mutex lock on do_transaction() function for SPI driver to avoid race
  condition.

v8 updates:
=========
- add multiple cards support.

v7 updates:
==========
- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
- function rte_pmd_i40e_set_switch_dev works as an external API,
  use port_id but not rte_eth_dev as parameter.
- add doxygen header here for the new API.
- update the rte_pmd_i40e_version.map.
- fix coding style issue.
- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
- for functions with a lot of similarity, extract out common function to reduce
  duplication.

v6 updates:
=========
- correct author information.
- correct typo in commit message and remove Gerrit Change-Id's before
  submitting upstream

v5 updates:
==========
- add lightweight fpga image support. in lightweight fpga image mode,
  ipn3ke representor will not be probed.

v4 updates:
===========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support

Andy Pei (2):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga: add lightweight fpga image support

Rosen Xu (3):
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei zhang (14):
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga/base: add secure support
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors
  raw/ifpga/base: add new API get board info
  raw/ifpga/base: add multiple cards support
  raw/ifpga: introducing new irq API

 config/common_base                             |   4 +-
 config/common_linux                            |   6 +
 drivers/meson.build                            |   6 +-
 drivers/net/i40e/base/i40e_type.h              |   3 +
 drivers/net/i40e/i40e_ethdev.c                 |   6 +
 drivers/net/i40e/rte_pmd_i40e.c                |  21 +
 drivers/net/i40e/rte_pmd_i40e.h                |  18 +
 drivers/net/i40e/rte_pmd_i40e_version.map      |   8 +-
 drivers/net/ipn3ke/Makefile                    |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c             | 289 +-------
 drivers/net/ipn3ke/ipn3ke_representor.c        |   8 +-
 drivers/net/ipn3ke/meson.build                 |  24 +-
 drivers/raw/ifpga/base/ifpga_api.c             |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h         |  75 +-
 drivers/raw/ifpga/base/ifpga_feature_dev.c     |  59 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h     |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c             | 166 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c       |  74 +-
 drivers/raw/ifpga/base/ifpga_hw.h              |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c            |  18 +
 drivers/raw/ifpga/base/ifpga_port_error.c      |  19 +
 drivers/raw/ifpga/base/meson.build             |   2 +-
 drivers/raw/ifpga/base/opae_debug.c            |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c           | 137 ++++
 drivers/raw/ifpga/base/opae_hw_api.h           |  26 +
 drivers/raw/ifpga/base/opae_i2c.c              |  44 +-
 drivers/raw/ifpga/base/opae_i2c.h              |   3 +-
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h     |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c      | 599 +++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h      | 157 ++++-
 drivers/raw/ifpga/base/opae_osdep.h            |   7 +-
 drivers/raw/ifpga/base/opae_spi.c              |   5 -
 drivers/raw/ifpga/base/opae_spi.h              |  26 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c  |  84 ++-
 drivers/raw/ifpga/ifpga_rawdev.c               | 903 ++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h               |  30 +
 drivers/raw/ifpga/meson.build                  |  25 +-
 drivers/raw/ifpga/rte_rawdev_ifpga_version.map |   6 +
 mk/rte.app.mk                                  |   2 +-
 39 files changed, 2451 insertions(+), 442 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v14 01/19] net/i40e: i40e support ipn3ke FPGA port bonding

[Andy Pei]

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h         |  3 +++
 drivers/net/i40e/i40e_ethdev.c            |  6 ++++++
 drivers/net/i40e/rte_pmd_i40e.c           | 21 +++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h           | 18 ++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e_version.map |  8 +++++++-
 5 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 77a4683..6c5c97c 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1392,6 +1392,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2903,6 +2906,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index 4c3c708..fdcb1a4 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -3207,3 +3207,24 @@ int rte_pmd_i40e_flow_add_del_packet_template(
 	I40E_WRITE_FLUSH(hw);
 	return 0;
 }
+
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev)
+{
+	struct rte_eth_dev *i40e_dev;
+	struct i40e_hw *hw;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+	i40e_dev = &rte_eth_devices[port_id];
+	if (!is_i40e_supported(i40e_dev))
+		return -ENOTSUP;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+	if (!hw)
+		return -1;
+
+	hw->switch_dev = switch_dev;
+
+	return 0;
+}
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..355b8be 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+/**
+ * For ipn3ke, i40e works with FPGA.
+ * In this situation, i40e get link status from fpga,
+ * fpga works as switch_dev for i40e.
+ * This function set switch_dev for i40e.
+ *
+ * @param inset
+ *    Input set value.
+ * @param field_idx
+ *    Field index for input set.
+ * @return
+ *   - (less than 0) if failed.
+ *   - (0) if success.
+ */
+__rte_experimental
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
index cccd576..79641f2 100644
--- a/drivers/net/i40e/rte_pmd_i40e_version.map
+++ b/drivers/net/i40e/rte_pmd_i40e_version.map
@@ -64,4 +64,10 @@ DPDK_18.02 {
 
 	rte_pmd_i40e_inset_get;
 	rte_pmd_i40e_inset_set;
-} DPDK_17.11;
\ No newline at end of file
+} DPDK_17.11;
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_i40e_set_switch_dev;
+} DPDK_18.02;
\ No newline at end of file
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 02/19] raw/ifpga/base: add irq support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME global error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

To build this feature, CONFIG_RTE_EAL_VFIO should be enabled.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                         |  2 +-
 config/common_linux                        |  6 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 59 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 19 ++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 18 +++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 19 ++++++++++
 6 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/config/common_base b/config/common_base
index b2be3d9..6cdc4fa 100644
--- a/config/common_base
+++ b/config/common_base
@@ -783,7 +783,7 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
 #
 # Compile PMD for Intel FPGA raw device
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
 
 #
 # Compile PMD for Intel IOAT raw device
diff --git a/config/common_linux b/config/common_linux
index 96e2e1f..a78b8c6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -68,3 +68,9 @@ CONFIG_RTE_LIBRTE_HINIC_PMD=y
 # Hisilicon HNS3 PMD driver
 #
 CONFIG_RTE_LIBRTE_HNS3_PMD=y
+
+#
+# Compile PMD for Intel FPGA raw device
+# To compile, CONFIG_RTE_EAL_VFIO should be enabled.
+#
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..0f852a7 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,61 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+#define FPGA_MAX_MSIX_VEC_COUNT	128
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MAX_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	/* 0 < irq_set->count < FPGA_MAX_MSIX_VEC_COUNT */
+	irq_set->count = count ?
+		(count > FPGA_MAX_MSIX_VEC_COUNT ?
+		 FPGA_MAX_MSIX_VEC_COUNT : count) : 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..2978c79 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,28 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&fme->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..c0aaf01 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,27 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..189f762 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,28 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 03/19] raw/ifpga/base: clear pending bit

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 2978c79..be041ec 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 04/19] raw/ifpga/base: add SEU error support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 31 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index be041ec..5d6d630 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,33 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr(struct ifpga_fme_hw *fme,
+		u64 *val, bool high)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+				FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	if (high)
+		*val = readq(&fme_err->seu_emr_h);
+	else
+		*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +297,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr(fme, &prop->data, 0);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr(fme, &prop->data, 1);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 05/19] raw/ifpga/base: add device tree support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

To eliminate build error, we currently do not compile raw/ifpga
and net/ipn3ke. User should install libfdt and libfdt-devel first,
modify config/common_linux, CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
to CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y, modify config/common_base,
CONFIG_RTE_LIBRTE_IPN3KE_PMD=n to CONFIG_RTE_LIBRTE_IPN3KE_PMD=y.
Then this function can work.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 config/common_base                        |   2 +-
 config/common_linux                       |   2 +-
 drivers/meson.build                       |   6 +-
 drivers/net/ipn3ke/meson.build            |  24 +++-
 drivers/raw/ifpga/base/meson.build        |   2 +-
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 drivers/raw/ifpga/meson.build             |  25 ++--
 mk/rte.app.mk                             |   2 +-
 9 files changed, 235 insertions(+), 21 deletions(-)

diff --git a/config/common_base b/config/common_base
index 6cdc4fa..66bdba1 100644
--- a/config/common_base
+++ b/config/common_base
@@ -342,7 +342,7 @@ CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
 #
 # Compile burst-oriented IPN3KE PMD driver
 #
-CONFIG_RTE_LIBRTE_IPN3KE_PMD=y
+CONFIG_RTE_LIBRTE_IPN3KE_PMD=n
 
 #
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
diff --git a/config/common_linux b/config/common_linux
index a78b8c6..c5cf3d6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -73,4 +73,4 @@ CONFIG_RTE_LIBRTE_HNS3_PMD=y
 # Compile PMD for Intel FPGA raw device
 # To compile, CONFIG_RTE_EAL_VFIO should be enabled.
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
diff --git a/drivers/meson.build b/drivers/meson.build
index 156d2dc..b8e969e 100644
--- a/drivers/meson.build
+++ b/drivers/meson.build
@@ -9,12 +9,12 @@ endif
 dpdk_driver_classes = ['common',
 	       'bus',
 	       'mempool', # depends on common and bus.
-	       'net',     # depends on common, bus and mempool.
+	       'raw',     # depends on common and bus.
+	       'net',     # depends on common, bus, mempool and raw.
 	       'crypto',  # depends on common, bus and mempool (net in future).
 	       'compress', # depends on common, bus, mempool.
 	       'event',   # depends on common, bus, mempool and net.
-	       'baseband', # depends on common and bus.
-	       'raw']     # depends on common, bus, mempool, net and event.
+	       'baseband'] # depends on common and bus.
 
 disabled_drivers = get_option('disable_drivers').split(',')
 
diff --git a/drivers/net/ipn3ke/meson.build b/drivers/net/ipn3ke/meson.build
index 74b4d7c..e3c8a67 100644
--- a/drivers/net/ipn3ke/meson.build
+++ b/drivers/net/ipn3ke/meson.build
@@ -8,10 +8,22 @@
 #  rte_eth_dev_destroy()
 #  rte_eth_switch_domain_free()
 #
-allow_experimental_apis = true
 
-sources += files('ipn3ke_ethdev.c',
-	'ipn3ke_representor.c',
-	'ipn3ke_tm.c',
-	'ipn3ke_flow.c')
-deps += ['bus_ifpga', 'sched']
+dep = dependency('libfdt', required: false)
+if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
+	build = false
+	reason = 'missing dependency, "libfdt"'
+endif
+
+if build
+	allow_experimental_apis = true
+
+	sources += files('ipn3ke_ethdev.c',
+		'ipn3ke_representor.c',
+		'ipn3ke_tm.c',
+		'ipn3ke_flow.c')
+	deps += ['bus_ifpga', 'sched', 'pmd_i40e', 'rawdev', 'rawdev_ifpga']
+endif
diff --git a/drivers/raw/ifpga/base/meson.build b/drivers/raw/ifpga/base/meson.build
index 69f6598..b13e13e 100644
--- a/drivers/raw/ifpga/base/meson.build
+++ b/drivers/raw/ifpga/base/meson.build
@@ -25,5 +25,5 @@ sources = [
 
 base_lib = static_library('ifpga_rawdev_base', sources,
 	dependencies: static_rte_eal,
-	c_args: c_args)
+	c_args: cflags)
 base_objs = base_lib.extract_all_objects()
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
index 0ab6fd7..69debff 100644
--- a/drivers/raw/ifpga/meson.build
+++ b/drivers/raw/ifpga/meson.build
@@ -3,18 +3,27 @@
 
 version = 1
 
-subdir('base')
-objs = [base_objs]
-
 dep = dependency('libfdt', required: false)
 if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
 	build = false
 	reason = 'missing dependency, "libfdt"'
 endif
-deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
-	'bus_vdev', 'bus_ifpga', 'net']
-sources = files('ifpga_rawdev.c')
 
-includes += include_directories('base')
+if build
+	subdir('base')
+	objs = [base_objs]
+
+	deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
+		'bus_vdev', 'bus_ifpga', 'net']
+	ext_deps += dep
 
-allow_experimental_apis = true
+	sources = files('ifpga_rawdev.c')
+
+	includes += include_directories('base')
+	includes += include_directories('../../net/ipn3ke')
+
+	allow_experimental_apis = true
+endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 1f5c748..934c06f 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -326,7 +326,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 06/19] raw/ifpga/base: align the send buffer for SPI

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 07/19] raw/ifpga/base: add sensor support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 08/19] raw/ifpga/base: introducing sensor APIs

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 09/19] raw/ifpga/base: update SEU register definition

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 10/19] raw/ifpga: add SEU error handler

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 245 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 1825143..f5e6119 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -27,6 +27,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -605,6 +607,236 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -652,6 +884,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -677,6 +910,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -696,6 +933,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -720,6 +958,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 11/19] raw/ifpga: add PCIe BDF devices tree scan

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 551 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 562 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index f5e6119..01ff76a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_malloc.h>
@@ -17,7 +19,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -25,6 +27,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -37,6 +40,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -64,6 +73,494 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(
+		struct ifpga_rawdev *ifpga_rdev, int force_disable);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+		int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
+					sensor->name, value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(
+	struct ifpga_rawdev *ifpga_rdev, int force_disable)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable || force_disable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev,
+					gsd_enable);
+				if (ret == 1 && !gsd_enable) {
+					gsd_enable = 1;
+					i = -1;
+				}
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -372,8 +869,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -842,6 +1340,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -855,7 +1354,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -869,6 +1368,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -987,6 +1494,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1018,13 +1526,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1038,7 +1565,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1065,6 +1593,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 12/19] net/ipn3ke: remove configuration for i40e port bonding

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/ipn3ke/Makefile                    |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c             | 289 +++----------------------
 drivers/net/ipn3ke/ipn3ke_representor.c        |   8 +-
 drivers/raw/ifpga/rte_rawdev_ifpga_version.map |   6 +
 4 files changed, 50 insertions(+), 255 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..2c65e49 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index 28d8aaf..3051cdf 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -324,7 +325,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -342,6 +344,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -409,9 +417,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -462,7 +467,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -489,7 +498,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -501,6 +515,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -508,6 +538,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -553,254 +584,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index d37f5e2..7e5d29d 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2918,8 +2919,11 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		rte_pmd_i40e_set_switch_dev(rpst->i40e_pf_eth_port_id,
+					    rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
diff --git a/drivers/raw/ifpga/rte_rawdev_ifpga_version.map b/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
index 9b9ab1a..73d3242 100644
--- a/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
+++ b/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
@@ -2,3 +2,9 @@ DPDK_18.05 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	ifpga_rawdev_get;
+} DPDK_18.05;
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 13/19] raw/ifpga/base: add secure support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 137 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 ++++++++++++-----
 4 files changed, 198 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..748ab56 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,63 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager")) {
+			sysmgr_offset = offset;
+			break;
+		}
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +576,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 14/19] raw/ifpga/base: configure FEC mode

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 15/19] raw/ifpga/base: clean fme errors

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occurred.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 5d6d630..5905eac 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 01ff76a..95f079a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1174,6 +1174,25 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1183,6 +1202,9 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 16/19] raw/ifpga/base: add new API get board info

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..9f0147d 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disaggregate;
+	u8 lightweight;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..799d67d 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweight = id.lightweiht;
+	fme->board_info.disaggregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweight:%s\n"
+			"disaggregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweight),
+			check_support(fme->board_info.disaggregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 17/19] raw/ifpga: add lightweight fpga image support

[Andy Pei]

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 95f079a..e87af66 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -835,6 +835,8 @@ static int set_surprise_link_check_aer(
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -861,22 +863,40 @@ static int set_surprise_link_check_aer(
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweight) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
-		uuid.b + 8, sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 18/19] raw/ifpga/base: add multiple cards support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, there is one MAX10 chip in each card, and
all of the sensors are connected to MAX10 chip. To support multiple
cards in one server, we introducing a sensor device list under
intel_max10_device instead of a global list. On the other hand, we
using separate intel_max10_device instance for each opae_adatper.

Add mutex lock on do_transaction() function for SPI driver to avoid
race condition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c            |  40 +++++++---
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          |  14 ++--
 drivers/raw/ifpga/base/opae_hw_api.h          |  15 ++--
 drivers/raw/ifpga/base/opae_i2c.c             |  44 ++++++++---
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_intel_max10.c     | 110 ++++++++++++++------------
 drivers/raw/ifpga/base/opae_intel_max10.h     |  19 +++--
 drivers/raw/ifpga/base/opae_spi.c             |   5 --
 drivers/raw/ifpga/base/opae_spi.h             |   3 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  44 +++++++++--
 drivers/raw/ifpga/ifpga_rawdev.c              |  14 ++--
 12 files changed, 205 insertions(+), 109 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 799d67d..c31a94c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -839,8 +839,13 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	struct ifpga_hw *hw;
 	u32 val;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
@@ -854,7 +859,10 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	fme->board_info.seu = id.seu;
 	fme->board_info.ptp = id.ptp;
 
-	dev_info(fme, "found: board: %s type: %s\n",
+	dev_info(fme, "found: PCI dev: %02x:%02x:%x board: %s type: %s\n",
+			hw->pci_data->bus,
+			hw->pci_data->devid,
+			hw->pci_data->function,
 			board_major_to_string(fme->board_info.major),
 			board_type_to_string(fme->board_info.type));
 
@@ -882,11 +890,11 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
-	if (max10_sys_read(MAX10_BUILD_VER, &val))
+	if (max10_sys_read(fme->max10_dev, MAX10_BUILD_VER, &val))
 		return -EINVAL;
 	fme->board_info.max10_version = val & 0xffffff;
 
-	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+	if (max10_sys_read(fme->max10_dev, NIOS2_FW_VERSION, &val))
 		return -EINVAL;
 	fme->board_info.nios_fw_version = val & 0xffffff;
 
@@ -897,12 +905,12 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	return 0;
 }
 
-static int spi_self_checking(void)
+static int spi_self_checking(struct intel_max10_device *dev)
 {
 	u32 val;
 	int ret;
 
-	ret = max10_sys_read(MAX10_TEST_REG, &val);
+	ret = max10_sys_read(dev, MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
@@ -937,10 +945,11 @@ static int fme_spi_init(struct ifpga_feature *feature)
 		goto spi_fail;
 	}
 
+
 	fme->max10_dev = max10;
 
 	/* SPI self test */
-	if (spi_self_checking()) {
+	if (spi_self_checking(max10)) {
 		ret = -EIO;
 		goto max10_fail;
 	}
@@ -1041,8 +1050,18 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 	struct ifpga_fme_hw *fme = (struct ifpga_fme_hw *)feature->parent;
 	struct altera_spi_device *spi_master;
 	struct intel_max10_device *max10;
+	struct ifpga_hw *hw;
+	struct opae_manager *mgr;
 	int ret = 0;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
+	mgr = hw->adapter->mgr;
+	if (!mgr)
+		return -ENODEV;
+
 	dev_info(fme, "FME SPI Master (NIOS) Init.\n");
 	dev_debug(fme, "FME SPI base addr %p.\n",
 			feature->addr);
@@ -1080,12 +1099,15 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 		goto release_dev;
 	}
 
+	max10->bus = hw->pci_data->bus;
+
 	fme_get_board_interface(fme);
 
 	fme->max10_dev = max10;
+	mgr->sensor_list = &max10->opae_sensor_list;
 
 	/* SPI self test */
-	if (spi_self_checking())
+	if (spi_self_checking(max10))
 		goto spi_fail;
 
 	return ret;
@@ -1344,7 +1366,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(dev, PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1372,7 +1394,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(sensor->value_reg, value)) {
+	if (max10_sys_read(dev, sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_debug.c b/drivers/raw/ifpga/base/opae_debug.c
index 88f2d5c..dad3ea3 100644
--- a/drivers/raw/ifpga/base/opae_debug.c
+++ b/drivers/raw/ifpga/base/opae_debug.c
@@ -59,6 +59,9 @@ static void opae_adapter_data_dump(void *data)
 		opae_log("OPAE Adapter Type = PCI\n");
 		opae_log("PCI Device ID: 0x%04x\n", d_pci->device_id);
 		opae_log("PCI Vendor ID: 0x%04x\n", d_pci->vendor_id);
+		opae_log("PCI bus: 0x%04x\n", d_pci->bus);
+		opae_log("PCI devid: 0x%04x\n", d_pci->devid);
+		opae_log("PCI function: 0x%04x\n", d_pci->function);
 
 		for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 			r = &d_pci->region[i];
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 1ccc967..c969dfe 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -583,11 +583,12 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_id(unsigned int id)
+opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (sensor->id == id)
 			return sensor;
 
@@ -601,11 +602,12 @@ struct opae_sensor_info *
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_name(const char *name)
+opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (!strcmp(sensor->name, name))
 			return sensor;
 
@@ -630,7 +632,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_name(name);
+	sensor = opae_mgr_get_sensor_by_name(mgr, name);
 	if (!sensor)
 		return -ENODEV;
 
@@ -658,7 +660,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_id(id);
+	sensor = opae_mgr_get_sensor_by_id(mgr, id);
 	if (!sensor)
 		return -ENODEV;
 
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index b78fbd5..cdf369f 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -40,6 +40,7 @@ struct opae_manager {
 	struct opae_adapter *adapter;
 	struct opae_manager_ops *ops;
 	struct opae_manager_networking_ops *network_ops;
+	struct opae_sensor_list *sensor_list;
 	void *data;
 };
 
@@ -75,9 +76,8 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
-extern struct opae_sensor_list opae_sensor_list;
-#define opae_mgr_for_each_sensor(sensor) \
-	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+#define opae_mgr_for_each_sensor(mgr, sensor) \
+	TAILQ_FOREACH(sensor, mgr->sensor_list, node)
 
 /* OPAE Manager APIs */
 struct opae_manager *
@@ -88,8 +88,10 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
-struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
-struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id);
 int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
 		const char *name, unsigned int *value);
 int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
@@ -241,6 +243,9 @@ struct opae_adapter_data_pci {
 	enum opae_adapter_type type;
 	u16 device_id;
 	u16 vendor_id;
+	u16 bus; /*Device bus for PCI */
+	u16 devid; /* Device ID */
+	u16 function; /* Device function */
 	struct opae_reg_region region[PCI_MAX_RESOURCE];
 	int vfio_dev_fd;  /* VFIO device file descriptor */
 };
diff --git a/drivers/raw/ifpga/base/opae_i2c.c b/drivers/raw/ifpga/base/opae_i2c.c
index f8bc247..846d751 100644
--- a/drivers/raw/ifpga/base/opae_i2c.c
+++ b/drivers/raw/ifpga/base/opae_i2c.c
@@ -28,6 +28,9 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 {
 	u8 msgbuf[2];
 	int i = 0;
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
 
 	if (flags & I2C_FLAG_ADDR16)
 		msgbuf[i++] = offset >> 8;
@@ -49,10 +52,16 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		},
 	};
 
-	if (!dev->xfer)
-		return -ENODEV;
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	ret = i2c_transfer(dev, msg, 2);
 
-	return i2c_transfer(dev, msg, 2);
+exit:
+	pthread_mutex_unlock(&dev->lock);
+	return ret;
 }
 
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
@@ -63,12 +72,18 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	int ret;
 	int i = 0;
 
-	if (!dev->xfer)
-		return -ENODEV;
+	pthread_mutex_lock(&dev->lock);
+
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
 
 	buf = opae_malloc(I2C_MAX_OFFSET_LEN + len);
-	if (!buf)
-		return -ENOMEM;
+	if (!buf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
 
 	msg.addr = slave_addr;
 	msg.flags = 0;
@@ -84,6 +99,8 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	ret = i2c_transfer(dev, &msg, 1);
 
 	opae_free(buf);
+exit:
+	pthread_mutex_unlock(&dev->lock);
 	return ret;
 }
 
@@ -477,14 +494,19 @@ struct altera_i2c_dev *altera_i2c_probe(void *base)
 	dev->i2c_clk = dev->i2c_param.ref_clk * 1000000;
 	dev->xfer = altera_i2c_xfer;
 
+	if (pthread_mutex_init(&dev->lock, NULL))
+		return NULL;
+
 	altera_i2c_hardware_init(dev);
 
 	return dev;
 }
 
-int altera_i2c_remove(struct altera_i2c_dev *dev)
+void altera_i2c_remove(struct altera_i2c_dev *dev)
 {
-	altera_i2c_disable(dev);
-
-	return 0;
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
+		altera_i2c_disable(dev);
+		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/base/opae_i2c.h b/drivers/raw/ifpga/base/opae_i2c.h
index 8890c8f..266e127 100644
--- a/drivers/raw/ifpga/base/opae_i2c.h
+++ b/drivers/raw/ifpga/base/opae_i2c.h
@@ -93,6 +93,7 @@ struct altera_i2c_dev {
 	u32 isr_mask;
 	u8 *buf;
 	int (*xfer)(struct altera_i2c_dev *dev, struct i2c_msg *msg, int num);
+	pthread_mutex_t lock;
 };
 
 /**
@@ -114,7 +115,7 @@ enum i2c_msg_flags {
 };
 
 struct altera_i2c_dev *altera_i2c_probe(void *base);
-int altera_i2c_remove(struct altera_i2c_dev *dev);
+void altera_i2c_remove(struct altera_i2c_dev *dev);
 int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		u32 offset, u8 *buf, u32 count);
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 748ab56..8e23ca1 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -5,45 +5,50 @@
 #include "opae_intel_max10.h"
 #include <libfdt.h>
 
-static struct intel_max10_device *g_max10;
-
-struct opae_sensor_list opae_sensor_list =
-	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
-
-int max10_reg_read(unsigned int reg, unsigned int *val)
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_read(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x\n", __func__, dev->bus, reg);
+
+	return spi_transaction_read(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)val);
 }
 
-int max10_reg_write(unsigned int reg, unsigned int val)
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val)
 {
 	unsigned int tmp = val;
 
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_write(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x, val:0x%x\n", __func__,
+			dev->bus, reg, val);
+
+	return spi_transaction_write(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)&tmp);
 }
 
-int max10_sys_read(unsigned int offset, unsigned int *val)
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_read(g_max10->base + offset, val);
+
+	return max10_reg_read(dev, dev->base + offset, val);
 }
 
-int max10_sys_write(unsigned int offset, unsigned int val)
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_write(g_max10->base + offset, val);
+	return max10_reg_write(dev, dev->base + offset, val);
 }
 
 static struct max10_compatible_id max10_id_table[] = {
@@ -86,8 +91,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
-static int altera_nor_flash_read(u32 offset,
-		void *buffer, u32 len)
+static int altera_nor_flash_read(struct intel_max10_device *dev,
+	u32 offset, void *buffer, u32 len)
 {
 	int word_len;
 	int i;
@@ -95,13 +100,13 @@ static int altera_nor_flash_read(u32 offset,
 	unsigned int value;
 	int ret;
 
-	if (!buffer || len <= 0)
+	if (!dev || !buffer || len <= 0)
 		return -ENODEV;
 
 	word_len = len/4;
 
 	for (i = 0; i < word_len; i++) {
-		ret = max10_reg_read(offset + i*4,
+		ret = max10_reg_read(dev, offset + i*4,
 				&value);
 		if (ret)
 			return -EBUSY;
@@ -112,12 +117,12 @@ static int altera_nor_flash_read(u32 offset,
 	return 0;
 }
 
-static int enable_nor_flash(bool on)
+static int enable_nor_flash(struct intel_max10_device *dev, bool on)
 {
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_sys_read(RSU_REG, &val);
+	ret = max10_sys_read(dev, RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -128,7 +133,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_sys_write(RSU_REG, val);
+	return max10_sys_write(dev, RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -140,7 +145,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_sys_read(DT_AVAIL_REG, &val);
+	ret = max10_sys_read(max10, DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -151,19 +156,19 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
+	ret = max10_sys_read(max10, DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
 	}
 
-	ret = enable_nor_flash(true);
+	ret = enable_nor_flash(max10, true);
 	if (ret) {
 		dev_err(max10 "fail to enable flash\n");
 		return ret;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	ret = altera_nor_flash_read(max10, dt_addr, &hdr, sizeof(hdr));
 	if (ret) {
 		dev_err(max10 "read fdt header fail\n");
 		goto done;
@@ -188,7 +193,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		goto done;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	ret = altera_nor_flash_read(max10, dt_addr, fdt_root, dt_size);
 	if (ret) {
 		dev_err(max10 "cannot read device table\n");
 		goto done;
@@ -207,7 +212,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	max10->fdt_root = fdt_root;
 
 done:
-	ret = enable_nor_flash(false);
+	ret = enable_nor_flash(max10, false);
 
 	if (ret && fdt_root)
 		opae_free(fdt_root);
@@ -298,12 +303,12 @@ static int fdt_get_named_reg(const void *fdt, int node, const char *name,
 	return fdt_get_reg(fdt, node, idx, start, size);
 }
 
-static void max10_sensor_uinit(void)
+static void max10_sensor_uinit(struct intel_max10_device *dev)
 {
 	struct opae_sensor_info *info;
 
-	TAILQ_FOREACH(info, &opae_sensor_list, node) {
-		TAILQ_REMOVE(&opae_sensor_list, info, node);
+	TAILQ_FOREACH(info, &dev->opae_sensor_list, node) {
+		TAILQ_REMOVE(&dev->opae_sensor_list, info, node);
 		opae_free(info);
 	}
 }
@@ -313,8 +318,8 @@ static bool sensor_reg_valid(struct sensor_reg *reg)
 	return !!reg->size;
 }
 
-static int max10_add_sensor(struct raw_sensor_info *info,
-		struct opae_sensor_info *sensor)
+static int max10_add_sensor(struct intel_max10_device *dev,
+	struct raw_sensor_info *info, struct opae_sensor_info *sensor)
 {
 	int i;
 	int ret = 0;
@@ -332,12 +337,15 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_sys_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(dev, info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
-		if (val == 0xdeadbeef)
+		if (val == 0xdeadbeef) {
+			dev_debug(dev, "%s: sensor:%s invalid 0x%x at:%d\n",
+				__func__, sensor->name, val, i);
 			continue;
+		}
 
 		val *= info->multiplier;
 
@@ -458,7 +466,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
 		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
 
-		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+		dev_debug(dev, "found sensor from DTB: %s: %s: %u: %u\n",
 				raw->name, raw->type,
 				raw->id, raw->multiplier);
 
@@ -473,15 +481,16 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 			goto free_sensor;
 		}
 
-		if (max10_add_sensor(raw, sensor)) {
+		if (max10_add_sensor(dev, raw, sensor)) {
 			ret = -EINVAL;
 			opae_free(sensor);
 			goto free_sensor;
 		}
 
-		if (sensor->flags & OPAE_SENSOR_VALID)
-			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
-		else
+		if (sensor->flags & OPAE_SENSOR_VALID) {
+			TAILQ_INSERT_TAIL(&dev->opae_sensor_list, sensor, node);
+			dev_info(dev, "found valid sensor: %s\n", sensor->name);
+		} else
 			opae_free(sensor);
 
 		opae_free(raw);
@@ -492,7 +501,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 free_sensor:
 	if (raw)
 		opae_free(raw);
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 	return ret;
 }
 
@@ -500,7 +509,7 @@ static int check_max10_version(struct intel_max10_device *dev)
 {
 	unsigned int v;
 
-	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+	if (!max10_reg_read(dev, MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
 				&v)) {
 		if (v != 0xffffffff) {
 			dev_info(dev, "secure MAX10 detected\n");
@@ -565,6 +574,8 @@ struct intel_max10_device *
 	if (!dev)
 		return NULL;
 
+	TAILQ_INIT(&dev->opae_sensor_list);
+
 	dev->spi_master = spi;
 
 	dev->spi_tran_dev = spi_transaction_init(spi, chipselect);
@@ -573,9 +584,6 @@ struct intel_max10_device *
 		goto free_dev;
 	}
 
-	/* set the max10 device firstly */
-	g_max10 = dev;
-
 	/* check the max10 version */
 	ret = check_max10_version(dev);
 	if (ret) {
@@ -601,7 +609,7 @@ struct intel_max10_device *
 	}
 
 	/* read FPGA loading information */
-	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
+	ret = max10_sys_read(dev, FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
 		goto release_max10_hw;
@@ -611,14 +619,13 @@ struct intel_max10_device *
 	return dev;
 
 release_max10_hw:
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return NULL;
@@ -629,7 +636,7 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
@@ -637,7 +644,6 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index e632941..123cdc4 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -26,6 +26,9 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SECURE		BIT(6)
 #define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
@@ -33,6 +36,8 @@ struct intel_max10_device {
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
 	unsigned int base; /* max10 base address */
+	u16 bus;
+	struct opae_sensor_list opae_sensor_list;
 };
 
 /* retimer speed */
@@ -136,17 +141,19 @@ struct opae_retimer_status {
 
 #define DFT_MAX_SIZE		0x7e0000
 
-int max10_reg_read(unsigned int reg, unsigned int *val);
-int max10_reg_write(unsigned int reg, unsigned int val);
-int max10_sys_read(unsigned int offset, unsigned int *val);
-int max10_sys_write(unsigned int offset, unsigned int val);
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val);
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val);
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val);
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
-/** List of opae sensors */
-TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
 
 #define SENSOR_REG_VALUE 0x0
 #define SENSOR_REG_HIGH_WARN 0x1
diff --git a/drivers/raw/ifpga/base/opae_spi.c b/drivers/raw/ifpga/base/opae_spi.c
index cc52782..bfdc83e 100644
--- a/drivers/raw/ifpga/base/opae_spi.c
+++ b/drivers/raw/ifpga/base/opae_spi.c
@@ -181,7 +181,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 	u32 rxd;
 	unsigned int tx_data;
 	u32 status;
-	int retry = 0;
 	int ret;
 
 	while (count < dev->len) {
@@ -194,10 +193,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 				return -EIO;
 			if (status & ALTERA_SPI_STATUS_RRDY_MSK)
 				break;
-			if (retry++ > SPI_MAX_RETRY) {
-				dev_err(dev, "%s, read timeout\n", __func__);
-				return -EBUSY;
-			}
 		}
 
 		ret = spi_reg_read(dev, ALTERA_SPI_RXDATA, &rxd);
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index 6355deb..d20a4c3 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -38,7 +38,7 @@
 #define SPI_WRITE 0x20
 #define WRITE_DATA_MASK GENMASK_ULL(31, 0)
 
-#define SPI_MAX_RETRY 100000
+#define SPI_MAX_RETRY 1000000
 
 #define TYPE_SPI 0
 #define TYPE_NIOS_SPI 1
@@ -102,6 +102,7 @@ struct spi_transaction_dev {
 	struct altera_spi_device *dev;
 	int chipselect;
 	struct spi_tran_buffer *buffer;
+	pthread_mutex_t lock;
 };
 
 struct spi_tran_header {
diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 06ca625..013efee 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -154,6 +154,8 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	unsigned int rx_len = 0;
 	int retry = 0;
 	int spi_flags;
+	unsigned long timeout = msecs_to_timer_cycles(1000);
+	unsigned long ticks;
 	unsigned int resp_max_len = 2 * resp_len;
 
 	print_buffer("before bytes:", send_data, send_len);
@@ -207,8 +209,11 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	if (ret != SPI_FOUND_EOP) {
 		tx_buffer = NULL;
 		tx_len = 0;
-		if (retry++ > 10) {
-			dev_err(NULL, "cannot found valid data from SPI\n");
+		ticks = rte_get_timer_cycles();
+		if (time_after(ticks, timeout) &&
+				retry++ > SPI_MAX_RETRY) {
+			dev_err(NULL, "Have retry %d, found invalid packet data\n",
+				retry);
 			return -EBUSY;
 		}
 
@@ -427,23 +432,36 @@ static int do_transaction(struct spi_transaction_dev *dev, unsigned int addr,
 int spi_transaction_read(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_READ : SPI_TRAN_NON_SEQ_READ);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 int spi_transaction_write(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_WRITE : SPI_TRAN_NON_SEQ_WRITE);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 		int chipselect)
 {
 	struct spi_transaction_dev *spi_tran_dev;
+	int ret;
 
 	spi_tran_dev = opae_malloc(sizeof(struct spi_transaction_dev));
 	if (!spi_tran_dev)
@@ -453,18 +471,28 @@ struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 	spi_tran_dev->chipselect = chipselect;
 
 	spi_tran_dev->buffer = opae_malloc(sizeof(struct spi_tran_buffer));
-	if (!spi_tran_dev->buffer) {
-		opae_free(spi_tran_dev);
-		return NULL;
+	if (!spi_tran_dev->buffer)
+		goto err;
+
+	ret = pthread_mutex_init(&spi_tran_dev->lock, NULL);
+	if (ret) {
+		dev_err(spi_tran_dev, "fail to init mutex lock\n");
+		goto err;
 	}
 
 	return spi_tran_dev;
+
+err:
+	opae_free(spi_tran_dev);
+	return NULL;
 }
 
 void spi_transaction_remove(struct spi_transaction_dev *dev)
 {
 	if (dev && dev->buffer)
 		opae_free(dev->buffer);
-	if (dev)
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
 		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index e87af66..7253644 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -360,7 +360,7 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 	if (!mgr)
 		return -ENODEV;
 
-	opae_mgr_for_each_sensor(sensor) {
+	opae_mgr_for_each_sensor(mgr, sensor) {
 		if (!(sensor->flags & OPAE_SENSOR_VALID))
 			goto fail;
 
@@ -369,8 +369,8 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 			goto fail;
 
 		if (value == 0xdeadbeef) {
-			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
-					sensor->name, value);
+			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value %x\n",
+					raw_dev->dev_id, sensor->name, value);
 			continue;
 		}
 
@@ -393,8 +393,9 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 		/* monitor 12V AUX sensor */
 		if (!strcmp(sensor->name, "12V AUX Voltage")) {
 			if (value < AUX_VOLTAGE_WARN) {
-				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
-						sensor->name, value);
+				IFPGA_RAWDEV_PMD_INFO(
+					"%s reach theshold %d mV\n",
+					sensor->name, value);
 				*gsd_start = true;
 				break;
 			}
@@ -1433,6 +1434,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->bus = pci_dev->addr.bus;
+	data->devid = pci_dev->addr.devid;
+	data->function = pci_dev->addr.function;
 	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
-- 
1.8.3.1

[dpdk-dev] [PATCH v14 19/19] raw/ifpga: introducing new irq API

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing new register and unregister API for ifpga interrupt.
1. register FME and AFU interrupt
   ifpga_register_msix_irq()

2. unregister FME and AFU interrupt
   ifpga_unregister_msix_irq()

On PAC N3000 card, there is one PCIe MSIX interrupt for FME managerment,
like the error report, thermal management, we use this interrupt in
ifpga_rawdev device driver. on the other hand, there are about 4 PCIe
MSIX interrupts are reserved for AFU which end-user can use those
interrupts in their AFU logic design. End-user can use those APIs to
register interrupt handler in their AFU drivers.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 105 +++++++++++++++++++++++++++------------
 drivers/raw/ifpga/ifpga_rawdev.h |  14 ++++++
 2 files changed, 88 insertions(+), 31 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 7253644..753dbad 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -78,6 +78,10 @@
 static int ifpga_monitor_start;
 static pthread_t ifpga_monitor_start_thread;
 
+#define IFPGA_MAX_IRQ 12
+/* 0 for FME interrupt, others are reserved for AFU irq */
+static struct rte_intr_handle ifpga_irq_handle[IFPGA_MAX_IRQ];
+
 static struct ifpga_rawdev *
 ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
 static int set_surprise_link_check_aer(
@@ -1331,53 +1335,90 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 	fme_err_handle_catfatal_error(mgr);
 }
 
-static struct rte_intr_handle fme_intr_handle;
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg)
+{
+	struct rte_intr_handle intr_handle;
+
+	if (type == IFPGA_FME_IRQ)
+		intr_handle = ifpga_irq_handle[0];
+	else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+	rte_intr_efd_disable(&intr_handle);
+
+	return rte_intr_callback_unregister(&intr_handle,
+			handler, arg);
+}
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg)
 {
 	int ret;
-	struct fpga_fme_err_irq_set err_irq_set;
+	struct rte_intr_handle intr_handle;
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_accelerator *acc;
 
-	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+	adapter = ifpga_rawdev_get_priv(dev);
+	if (!adapter)
+		return -ENODEV;
 
-	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
-	if (ret)
-		return -EINVAL;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
 
-	fme_intr_handle.fd = fme_intr_handle.efds[0];
+	if (type == IFPGA_FME_IRQ) {
+		intr_handle = ifpga_irq_handle[0];
+		count = 1;
+	} else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
-			fme_intr_handle.vfio_dev_fd,
-			fme_intr_handle.efds[0], fme_intr_handle.fd);
+	intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
 
-	err_irq_set.evtfd = fme_intr_handle.efds[0];
-	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	ret = rte_intr_efd_enable(&intr_handle, count);
 	if (ret)
-		return -EINVAL;
+		return -ENODEV;
+
+	intr_handle.fd = intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("register %s irq, vfio_fd=%d, fd=%d\n",
+			name, intr_handle.vfio_dev_fd,
+			intr_handle.fd);
+
+	if (type == IFPGA_FME_IRQ) {
+		struct fpga_fme_err_irq_set err_irq_set;
+		err_irq_set.evtfd = intr_handle.efds[0];
+
+		ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+		if (ret)
+			return -EINVAL;
+	} else if (type == IFPGA_AFU_IRQ) {
+		acc = opae_adapter_get_acc(adapter, port_id);
+		if (!acc)
+			return -EINVAL;
 
-	/* register FME interrupt using DPDK API */
-	ret = rte_intr_callback_register(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
+		ret = opae_acc_set_irq(acc, vec_start, count, intr_handle.efds);
+		if (ret)
+			return -EINVAL;
+	}
+
+	/* register interrupt handler using DPDK API */
+	ret = rte_intr_callback_register(&intr_handle,
+			handler, (void *)arg);
 	if (ret)
 		return -EINVAL;
 
-	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+	IFPGA_RAWDEV_PMD_INFO("success register %s interrupt\n", name);
 
 	return 0;
 }
 
 static int
-ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
-{
-	rte_intr_efd_disable(&fme_intr_handle);
-
-	return rte_intr_callback_unregister(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
-}
-
-static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -1463,7 +1504,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
-	ret = ifpga_register_fme_interrupt(mgr);
+	ret = ifpga_register_msix_irq(rawdev, 0, IFPGA_FME_IRQ, 0, 0,
+			fme_interrupt_handler, "fme_irq", mgr);
 	if (ret)
 		goto free_adapter_data;
 
@@ -1515,7 +1557,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	if (!mgr)
 		return -ENODEV;
 
-	if (ifpga_unregister_fme_interrupt(mgr))
+	if (ifpga_unregister_msix_irq(IFPGA_FME_IRQ, 0,
+				fme_interrupt_handler, mgr))
 		return -EINVAL;
 
 	opae_adapter_data_free(adapter->data);
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index bd42083..7754beb 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -62,4 +62,18 @@ struct ifpga_rawdev {
 struct ifpga_rawdev *
 ifpga_rawdev_get(const struct rte_rawdev *rawdev);
 
+enum ifpga_irq_type {
+	IFPGA_FME_IRQ = 0,
+	IFPGA_AFU_IRQ = 1,
+};
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg);
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v14 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Ye Xiaolong]

On 10/28, Andy Pei wrote:
>This patch set adds PCIe AER disable and FPGA interrupt support for
>ipn3ke. It also provides a small rework for port bonding between FPGA
>line side port and I40e PF port.
>
>What is the PCI Express AER(Advanced Error Reporting)?
>Advanced Error Reporting capability is implemented with a PCI Express
>advanced error reporting extended capability structure providing more
>robust error reporting. It's also one of PCI Express error reporting
>paradigms. AER is supported by most of PCIe devices.
>
>In PAC N3000 card, some uncertainty errors will cause FPGA reload,
>such as temperature is higher than threshold. From Software point of
>view, FPGA reload means FPGA unplug and plug. For avoiding system
>crash we need to clear AER register before these errors occur.
>
>Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga
>rawdev driver, we implement a FME interrupt function to notify errors
>reported by FME. Besides this, OPAE share code also provide a common
>AFU interrupt API for users to register their own interrupt functions.
>

Series applied to dpdk-next-net-intel. Thanks.

Re: [dpdk-dev] [PATCH v14 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Xu, Rosen]

Thanks Xiaolong.

> -----Original Message-----
> From: Ye, Xiaolong
> Sent: Thursday, October 31, 2019 10:01
> To: Pei, Andy <andy.pei@intel.com>
> Cc: dev@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> <tianfei.zhang@intel.com>; Yigit, Ferruh <ferruh.yigit@intel.com>
> Subject: Re: [PATCH v14 00/19] add PCIe AER disable and IRQ support for
> ipn3ke
> 
> On 10/28, Andy Pei wrote:
> >This patch set adds PCIe AER disable and FPGA interrupt support for
> >ipn3ke. It also provides a small rework for port bonding between FPGA
> >line side port and I40e PF port.
> >
> >What is the PCI Express AER(Advanced Error Reporting)?
> >Advanced Error Reporting capability is implemented with a PCI Express
> >advanced error reporting extended capability structure providing more
> >robust error reporting. It's also one of PCI Express error reporting
> >paradigms. AER is supported by most of PCIe devices.
> >
> >In PAC N3000 card, some uncertainty errors will cause FPGA reload, such
> >as temperature is higher than threshold. From Software point of view,
> >FPGA reload means FPGA unplug and plug. For avoiding system crash we
> >need to clear AER register before these errors occur.
> >
> >Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga
> >rawdev driver, we implement a FME interrupt function to notify errors
> >reported by FME. Besides this, OPAE share code also provide a common
> >AFU interrupt API for users to register their own interrupt functions.
> >
> 
> Series applied to dpdk-next-net-intel. Thanks.

Re: [dpdk-dev] [PATCH v14 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Pei, Andy]

Thanks Xiaolong.

-----Original Message-----
From: Ye, Xiaolong 
Sent: Thursday, October 31, 2019 10:01 AM
To: Pei, Andy <andy.pei@intel.com>
Cc: dev@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>; Yigit, Ferruh <ferruh.yigit@intel.com>
Subject: Re: [PATCH v14 00/19] add PCIe AER disable and IRQ support for ipn3ke

On 10/28, Andy Pei wrote:
>This patch set adds PCIe AER disable and FPGA interrupt support for 
>ipn3ke. It also provides a small rework for port bonding between FPGA 
>line side port and I40e PF port.
>
>What is the PCI Express AER(Advanced Error Reporting)?
>Advanced Error Reporting capability is implemented with a PCI Express 
>advanced error reporting extended capability structure providing more 
>robust error reporting. It's also one of PCI Express error reporting 
>paradigms. AER is supported by most of PCIe devices.
>
>In PAC N3000 card, some uncertainty errors will cause FPGA reload, such 
>as temperature is higher than threshold. From Software point of view, 
>FPGA reload means FPGA unplug and plug. For avoiding system crash we 
>need to clear AER register before these errors occur.
>
>Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga 
>rawdev driver, we implement a FME interrupt function to notify errors 
>reported by FME. Besides this, OPAE share code also provide a common 
>AFU interrupt API for users to register their own interrupt functions.
>

Series applied to dpdk-next-net-intel. Thanks.

[dpdk-dev] [PATCH v15 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Rosen Xu]

This patch set adds PCIe AER disable and FPGA interrupt support for
ipn3ke. It also provides a small rework for port bonding between FPGA
line side port and I40e PF port.

What is the PCI Express AER(Advanced Error Reporting)?
Advanced Error Reporting capability is implemented with a PCI Express
advanced error reporting extended capability structure providing more
robust error reporting. It's also one of PCI Express error reporting
paradigms. AER is supported by most of PCIe devices.

In PAC N3000 card, some uncertainty errors will cause FPGA reload,
such as temperature is higher than threshold. From Software point of
view, FPGA reload means FPGA unplug and plug. For avoiding system
crash we need to clear AER register before these errors occur.

Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga
rawdev driver, we implement a FME interrupt function to notify errors
reported by FME. Besides this, OPAE share code also provide a common
AFU interrupt API for users to register their own interrupt functions.

v15 updates:
=========
- fix share library build issues.

v14 updates:
=========
- fix coding style and remove unnecessary comments.

v13 updates:
=========
- fix meson build issue.

v12 updates:
=========
- fix meson build issue.
 
v11 updates:
=========
- move symbol ifpga_rawdev_ge to the EXPERIMENTAL section of 
  drivers/raw/ifpga/rte_rawdev_ifpga_version.map

v10 updates:
=========
- introducing new irq API
- fix meson build issue

v9 updates:
=========
- Add mutex lock on do_transaction() function for SPI driver to avoid race
  condition.

v8 updates:
=========
- add multiple cards support.

v7 updates:
==========
- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
- function rte_pmd_i40e_set_switch_dev works as an external API,
  use port_id but not rte_eth_dev as parameter.
- add doxygen header here for the new API.
- update the rte_pmd_i40e_version.map.
- fix coding style issue.
- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
- for functions with a lot of similarity, extract out common function to reduce
  duplication.

v6 updates:
=========
- correct author information.
- correct typo in commit message and remove Gerrit Change-Id's before
  submitting upstream

v5 updates:
==========
- add lightweight fpga image support. in lightweight fpga image mode,
  ipn3ke representor will not be probed.

v4 updates:
===========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support

Andy Pei (2):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga: add lightweight fpga image support

Rosen Xu (3):
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei zhang (14):
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga/base: add secure support
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors
  raw/ifpga/base: add new API get board info
  raw/ifpga/base: add multiple cards support
  raw/ifpga: introducing new irq API

 config/common_base                            |   4 +-
 config/common_linux                           |   6 +
 drivers/meson.build                           |   6 +-
 drivers/net/i40e/base/i40e_type.h             |   3 +
 drivers/net/i40e/i40e_ethdev.c                |   6 +
 drivers/net/i40e/rte_pmd_i40e.c               |  21 +
 drivers/net/i40e/rte_pmd_i40e.h               |  18 +
 drivers/net/i40e/rte_pmd_i40e_version.map     |   8 +-
 drivers/net/ipn3ke/Makefile                   |   1 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 293 ++-------
 drivers/net/ipn3ke/ipn3ke_flow.c              |   6 +
 drivers/net/ipn3ke/ipn3ke_rawdev_api.h        |  12 +
 drivers/net/ipn3ke/ipn3ke_representor.c       |  10 +-
 drivers/net/ipn3ke/meson.build                |  24 +-
 drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map |   6 +
 drivers/raw/ifpga/Makefile                    |   5 +
 drivers/raw/ifpga/base/ifpga_api.c            |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h        |  75 ++-
 drivers/raw/ifpga/base/ifpga_feature_dev.c    |  59 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c            | 166 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c      |  74 ++-
 drivers/raw/ifpga/base/ifpga_hw.h             |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c           |  18 +
 drivers/raw/ifpga/base/ifpga_port_error.c     |  19 +
 drivers/raw/ifpga/base/meson.build            |   2 +-
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          | 137 ++++
 drivers/raw/ifpga/base/opae_hw_api.h          |  26 +
 drivers/raw/ifpga/base/opae_i2c.c             |  44 +-
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c     | 599 ++++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h     | 157 ++++-
 drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
 drivers/raw/ifpga/base/opae_spi.c             |   5 -
 drivers/raw/ifpga/base/opae_spi.h             |  26 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  84 ++-
 drivers/raw/ifpga/ifpga_rawdev.c              | 907 +++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h              |  30 +
 drivers/raw/ifpga/meson.build                 |  26 +-
 mk/rte.app.mk                                 |   2 +-
 42 files changed, 2484 insertions(+), 442 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v15 01/19] net/i40e: i40e support ipn3ke FPGA port bonding

[Rosen Xu]

From: Andy Pei <andy.pei@intel.com>

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h         |  3 +++
 drivers/net/i40e/i40e_ethdev.c            |  6 ++++++
 drivers/net/i40e/rte_pmd_i40e.c           | 21 +++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h           | 18 ++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e_version.map |  8 +++++++-
 5 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 77a4683..6c5c97c 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1392,6 +1392,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2903,6 +2906,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index 4c3c708..fdcb1a4 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -3207,3 +3207,24 @@ int rte_pmd_i40e_flow_add_del_packet_template(
 	I40E_WRITE_FLUSH(hw);
 	return 0;
 }
+
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev)
+{
+	struct rte_eth_dev *i40e_dev;
+	struct i40e_hw *hw;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+	i40e_dev = &rte_eth_devices[port_id];
+	if (!is_i40e_supported(i40e_dev))
+		return -ENOTSUP;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+	if (!hw)
+		return -1;
+
+	hw->switch_dev = switch_dev;
+
+	return 0;
+}
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..355b8be 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+/**
+ * For ipn3ke, i40e works with FPGA.
+ * In this situation, i40e get link status from fpga,
+ * fpga works as switch_dev for i40e.
+ * This function set switch_dev for i40e.
+ *
+ * @param inset
+ *    Input set value.
+ * @param field_idx
+ *    Field index for input set.
+ * @return
+ *   - (less than 0) if failed.
+ *   - (0) if success.
+ */
+__rte_experimental
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
index cccd576..79641f2 100644
--- a/drivers/net/i40e/rte_pmd_i40e_version.map
+++ b/drivers/net/i40e/rte_pmd_i40e_version.map
@@ -64,4 +64,10 @@ DPDK_18.02 {
 
 	rte_pmd_i40e_inset_get;
 	rte_pmd_i40e_inset_set;
-} DPDK_17.11;
\ No newline at end of file
+} DPDK_17.11;
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_i40e_set_switch_dev;
+} DPDK_18.02;
\ No newline at end of file
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 02/19] raw/ifpga/base: add irq support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME global error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

To build this feature, CONFIG_RTE_EAL_VFIO should be enabled.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                         |  2 +-
 config/common_linux                        |  6 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 59 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 19 ++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 18 +++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 19 ++++++++++
 6 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/config/common_base b/config/common_base
index 1858598..86c770b 100644
--- a/config/common_base
+++ b/config/common_base
@@ -783,7 +783,7 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
 #
 # Compile PMD for Intel FPGA raw device
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
 
 #
 # Compile PMD for Intel IOAT raw device
diff --git a/config/common_linux b/config/common_linux
index 96e2e1f..a78b8c6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -68,3 +68,9 @@ CONFIG_RTE_LIBRTE_HINIC_PMD=y
 # Hisilicon HNS3 PMD driver
 #
 CONFIG_RTE_LIBRTE_HNS3_PMD=y
+
+#
+# Compile PMD for Intel FPGA raw device
+# To compile, CONFIG_RTE_EAL_VFIO should be enabled.
+#
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..0f852a7 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,61 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+#define FPGA_MAX_MSIX_VEC_COUNT	128
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MAX_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	/* 0 < irq_set->count < FPGA_MAX_MSIX_VEC_COUNT */
+	irq_set->count = count ?
+		(count > FPGA_MAX_MSIX_VEC_COUNT ?
+		 FPGA_MAX_MSIX_VEC_COUNT : count) : 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..2978c79 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,28 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&fme->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..c0aaf01 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,27 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..189f762 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,28 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 03/19] raw/ifpga/base: clear pending bit

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 2978c79..be041ec 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 04/19] raw/ifpga/base: add SEU error support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 31 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index be041ec..5d6d630 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,33 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr(struct ifpga_fme_hw *fme,
+		u64 *val, bool high)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+				FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	if (high)
+		*val = readq(&fme_err->seu_emr_h);
+	else
+		*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +297,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr(fme, &prop->data, 0);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr(fme, &prop->data, 1);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 05/19] raw/ifpga/base: add device tree support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

To eliminate build error, we currently do not compile raw/ifpga
and net/ipn3ke. User should install libfdt and libfdt-devel first,
modify config/common_linux, CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
to CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y, modify config/common_base,
CONFIG_RTE_LIBRTE_IPN3KE_PMD=n to CONFIG_RTE_LIBRTE_IPN3KE_PMD=y.
Then this function can work.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 config/common_base                        |   2 +-
 config/common_linux                       |   2 +-
 drivers/meson.build                       |   6 +-
 drivers/net/ipn3ke/meson.build            |  24 +++-
 drivers/raw/ifpga/base/meson.build        |   2 +-
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 drivers/raw/ifpga/meson.build             |  25 ++--
 mk/rte.app.mk                             |   2 +-
 9 files changed, 235 insertions(+), 21 deletions(-)

diff --git a/config/common_base b/config/common_base
index 86c770b..0b02ebf 100644
--- a/config/common_base
+++ b/config/common_base
@@ -342,7 +342,7 @@ CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
 #
 # Compile burst-oriented IPN3KE PMD driver
 #
-CONFIG_RTE_LIBRTE_IPN3KE_PMD=y
+CONFIG_RTE_LIBRTE_IPN3KE_PMD=n
 
 #
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
diff --git a/config/common_linux b/config/common_linux
index a78b8c6..c5cf3d6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -73,4 +73,4 @@ CONFIG_RTE_LIBRTE_HNS3_PMD=y
 # Compile PMD for Intel FPGA raw device
 # To compile, CONFIG_RTE_EAL_VFIO should be enabled.
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
diff --git a/drivers/meson.build b/drivers/meson.build
index 156d2dc..b8e969e 100644
--- a/drivers/meson.build
+++ b/drivers/meson.build
@@ -9,12 +9,12 @@ endif
 dpdk_driver_classes = ['common',
 	       'bus',
 	       'mempool', # depends on common and bus.
-	       'net',     # depends on common, bus and mempool.
+	       'raw',     # depends on common and bus.
+	       'net',     # depends on common, bus, mempool and raw.
 	       'crypto',  # depends on common, bus and mempool (net in future).
 	       'compress', # depends on common, bus, mempool.
 	       'event',   # depends on common, bus, mempool and net.
-	       'baseband', # depends on common and bus.
-	       'raw']     # depends on common, bus, mempool, net and event.
+	       'baseband'] # depends on common and bus.
 
 disabled_drivers = get_option('disable_drivers').split(',')
 
diff --git a/drivers/net/ipn3ke/meson.build b/drivers/net/ipn3ke/meson.build
index 74b4d7c..e3c8a67 100644
--- a/drivers/net/ipn3ke/meson.build
+++ b/drivers/net/ipn3ke/meson.build
@@ -8,10 +8,22 @@
 #  rte_eth_dev_destroy()
 #  rte_eth_switch_domain_free()
 #
-allow_experimental_apis = true
 
-sources += files('ipn3ke_ethdev.c',
-	'ipn3ke_representor.c',
-	'ipn3ke_tm.c',
-	'ipn3ke_flow.c')
-deps += ['bus_ifpga', 'sched']
+dep = dependency('libfdt', required: false)
+if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
+	build = false
+	reason = 'missing dependency, "libfdt"'
+endif
+
+if build
+	allow_experimental_apis = true
+
+	sources += files('ipn3ke_ethdev.c',
+		'ipn3ke_representor.c',
+		'ipn3ke_tm.c',
+		'ipn3ke_flow.c')
+	deps += ['bus_ifpga', 'sched', 'pmd_i40e', 'rawdev', 'rawdev_ifpga']
+endif
diff --git a/drivers/raw/ifpga/base/meson.build b/drivers/raw/ifpga/base/meson.build
index 69f6598..b13e13e 100644
--- a/drivers/raw/ifpga/base/meson.build
+++ b/drivers/raw/ifpga/base/meson.build
@@ -25,5 +25,5 @@ sources = [
 
 base_lib = static_library('ifpga_rawdev_base', sources,
 	dependencies: static_rte_eal,
-	c_args: c_args)
+	c_args: cflags)
 base_objs = base_lib.extract_all_objects()
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
index 0ab6fd7..69debff 100644
--- a/drivers/raw/ifpga/meson.build
+++ b/drivers/raw/ifpga/meson.build
@@ -3,18 +3,27 @@
 
 version = 1
 
-subdir('base')
-objs = [base_objs]
-
 dep = dependency('libfdt', required: false)
 if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
 	build = false
 	reason = 'missing dependency, "libfdt"'
 endif
-deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
-	'bus_vdev', 'bus_ifpga', 'net']
-sources = files('ifpga_rawdev.c')
 
-includes += include_directories('base')
+if build
+	subdir('base')
+	objs = [base_objs]
+
+	deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
+		'bus_vdev', 'bus_ifpga', 'net']
+	ext_deps += dep
 
-allow_experimental_apis = true
+	sources = files('ifpga_rawdev.c')
+
+	includes += include_directories('base')
+	includes += include_directories('../../net/ipn3ke')
+
+	allow_experimental_apis = true
+endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 059fe5d..0514249 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -328,7 +328,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 06/19] raw/ifpga/base: align the send buffer for SPI

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 07/19] raw/ifpga/base: add sensor support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 08/19] raw/ifpga/base: introducing sensor APIs

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 09/19] raw/ifpga/base: update SEU register definition

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 10/19] raw/ifpga: add SEU error handler

[Rosen Xu]

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 245 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 1825143..f5e6119 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -27,6 +27,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -605,6 +607,236 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -652,6 +884,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -677,6 +910,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -696,6 +933,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -720,6 +958,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 11/19] raw/ifpga: add PCIe BDF devices tree scan

[Rosen Xu]

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 551 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 562 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index f5e6119..01ff76a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_malloc.h>
@@ -17,7 +19,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -25,6 +27,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -37,6 +40,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -64,6 +73,494 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(
+		struct ifpga_rawdev *ifpga_rdev, int force_disable);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+		int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
+					sensor->name, value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(
+	struct ifpga_rawdev *ifpga_rdev, int force_disable)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable || force_disable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev,
+					gsd_enable);
+				if (ret == 1 && !gsd_enable) {
+					gsd_enable = 1;
+					i = -1;
+				}
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -372,8 +869,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -842,6 +1340,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -855,7 +1354,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -869,6 +1368,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -987,6 +1494,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1018,13 +1526,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1038,7 +1565,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1065,6 +1593,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 12/19] net/ipn3ke: remove configuration for i40e port bonding

[Rosen Xu]

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/ipn3ke/Makefile                   |   1 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 293 ++++----------------------
 drivers/net/ipn3ke/ipn3ke_flow.c              |   6 +
 drivers/net/ipn3ke/ipn3ke_rawdev_api.h        |  12 ++
 drivers/net/ipn3ke/ipn3ke_representor.c       |  10 +-
 drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map |   6 +
 drivers/raw/ifpga/Makefile                    |   5 +
 drivers/raw/ifpga/ifpga_rawdev.c              |   4 +
 drivers/raw/ifpga/meson.build                 |   5 +-
 9 files changed, 85 insertions(+), 257 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..8c58d71 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,7 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index 28d8aaf..5b5510f 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -35,6 +36,8 @@
 	{ 0, 0 /* sentinel */ },
 };
 
+struct ipn3ke_pub_func ipn3ke_bridge_func;
+
 static int
 ipn3ke_indirect_read(struct ipn3ke_hw *hw, uint32_t *rd_data,
 	uint32_t addr, uint32_t dev_sel, uint32_t eth_group_sel)
@@ -324,7 +327,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -342,6 +346,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -409,9 +419,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -462,7 +469,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -489,7 +500,14 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	if (ipn3ke_bridge_func.get_ifpga_rawdev == NULL)
+		return -ENOMEM;
+	ifpga_dev = ipn3ke_bridge_func.get_ifpga_rawdev(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -501,6 +519,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -508,6 +542,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -553,254 +588,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_flow.c b/drivers/net/ipn3ke/ipn3ke_flow.c
index 9fc3c8b..f857e64 100644
--- a/drivers/net/ipn3ke/ipn3ke_flow.c
+++ b/drivers/net/ipn3ke/ipn3ke_flow.c
@@ -18,6 +18,12 @@
 #include <rte_malloc.h>
 #include <rte_eth_ctrl.h>
 #include <rte_tailq.h>
+#include <rte_rawdev.h>
+#include <rte_rawdev_pmd.h>
+#include <rte_bus_ifpga.h>
+#include <ifpga_common.h>
+#include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
diff --git a/drivers/net/ipn3ke/ipn3ke_rawdev_api.h b/drivers/net/ipn3ke/ipn3ke_rawdev_api.h
index 671fae8..fd2393f 100644
--- a/drivers/net/ipn3ke/ipn3ke_rawdev_api.h
+++ b/drivers/net/ipn3ke/ipn3ke_rawdev_api.h
@@ -59,4 +59,16 @@ struct ifpga_rawdevg_link_info {
 	enum ifpga_rawdev_link_speed link_speed;
 };
 
+struct ipn3ke_pub_func {
+	struct ifpga_rawdev *(*get_ifpga_rawdev)(const struct rte_rawdev *rdv);
+	int (*set_i40e_sw_dev)(uint16_t port_id, struct rte_eth_dev *sw_dev);
+};
+
+/**
+ * @internal
+ * The publid functions of bridge PAC N3000 FPGA and I40e.
+ */
+extern struct ipn3ke_pub_func ipn3ke_bridge_func;
+
+
 #endif /* _IFPGA_RAWDEV_H_ */
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index d37f5e2..8d9ebef 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -2914,12 +2914,18 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	if (representor_param->port_id >= representor_param->hw->port_num)
 		return -ENODEV;
 
+	if (ipn3ke_bridge_func.set_i40e_sw_dev == NULL)
+		return -ENOMEM;
+
 	rpst->ethdev = ethdev;
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		ipn3ke_bridge_func.set_i40e_sw_dev(rpst->i40e_pf_eth_port_id,
+					    rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
diff --git a/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map b/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
index fc8c95e..fbb74ee 100644
--- a/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
+++ b/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
@@ -2,3 +2,9 @@ DPDK_19.05 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	ipn3ke_bridge_func;
+} DPDK_19.05;
diff --git a/drivers/raw/ifpga/Makefile b/drivers/raw/ifpga/Makefile
index 655b292..8fcdb09 100644
--- a/drivers/raw/ifpga/Makefile
+++ b/drivers/raw/ifpga/Makefile
@@ -13,6 +13,7 @@ CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
 CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga_rawdev
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 CFLAGS += -I$(RTE_SDK)/drivers/net/ipn3ke
 LDLIBS += -lrte_eal
 LDLIBS += -lrte_rawdev
@@ -20,6 +21,10 @@ LDLIBS += -lrte_bus_vdev
 LDLIBS += -lrte_kvargs
 LDLIBS += -lrte_bus_pci
 LDLIBS += -lrte_bus_ifpga
+LDLIBS += -lpthread
+LDLIBS += -lfdt
+LDLIBS += -lrte_pmd_i40e
+LDLIBS += -lrte_pmd_ipn3ke
 
 EXPORT_MAP := rte_rawdev_ifpga_version.map
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 01ff76a..b79403e 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -28,6 +28,7 @@
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
 #include <rte_string_fns.h>
+#include <rte_pmd_i40e.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -1368,6 +1369,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	ipn3ke_bridge_func.get_ifpga_rawdev = ifpga_rawdev_get;
+	ipn3ke_bridge_func.set_i40e_sw_dev = rte_pmd_i40e_set_switch_dev;
+
 	dev = ifpga_rawdev_allocate(rawdev);
 	if (dev == NULL) {
 		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
index 69debff..c3459e2 100644
--- a/drivers/raw/ifpga/meson.build
+++ b/drivers/raw/ifpga/meson.build
@@ -16,14 +16,15 @@ if build
 	subdir('base')
 	objs = [base_objs]
 
-	deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
-		'bus_vdev', 'bus_ifpga', 'net']
+	deps += ['ethdev', 'rawdev', 'pci', 'bus_pci', 'kvargs',
+		'bus_vdev', 'bus_ifpga', 'net', 'i40e', 'ipn3ke']
 	ext_deps += dep
 
 	sources = files('ifpga_rawdev.c')
 
 	includes += include_directories('base')
 	includes += include_directories('../../net/ipn3ke')
+	includes += include_directories('../../net/i40e')
 
 	allow_experimental_apis = true
 endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 13/19] raw/ifpga/base: add secure support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 137 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 ++++++++++++-----
 4 files changed, 198 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..748ab56 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,63 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager")) {
+			sysmgr_offset = offset;
+			break;
+		}
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +576,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 14/19] raw/ifpga/base: configure FEC mode

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 15/19] raw/ifpga/base: clean fme errors

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occurred.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 5d6d630..5905eac 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index b79403e..9c9da58 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1175,6 +1175,25 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1184,6 +1203,9 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 16/19] raw/ifpga/base: add new API get board info

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..9f0147d 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disaggregate;
+	u8 lightweight;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..799d67d 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweight = id.lightweiht;
+	fme->board_info.disaggregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweight:%s\n"
+			"disaggregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweight),
+			check_support(fme->board_info.disaggregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 17/19] raw/ifpga: add lightweight fpga image support

[Rosen Xu]

From: Andy Pei <andy.pei@intel.com>

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 9c9da58..5701e2d 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -836,6 +836,8 @@ static int set_surprise_link_check_aer(
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -862,22 +864,40 @@ static int set_surprise_link_check_aer(
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweight) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
-		uuid.b + 8, sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 18/19] raw/ifpga/base: add multiple cards support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, there is one MAX10 chip in each card, and
all of the sensors are connected to MAX10 chip. To support multiple
cards in one server, we introducing a sensor device list under
intel_max10_device instead of a global list. On the other hand, we
using separate intel_max10_device instance for each opae_adatper.

Add mutex lock on do_transaction() function for SPI driver to avoid
race condition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c            |  40 +++++++---
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          |  14 ++--
 drivers/raw/ifpga/base/opae_hw_api.h          |  15 ++--
 drivers/raw/ifpga/base/opae_i2c.c             |  44 ++++++++---
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_intel_max10.c     | 110 ++++++++++++++------------
 drivers/raw/ifpga/base/opae_intel_max10.h     |  19 +++--
 drivers/raw/ifpga/base/opae_spi.c             |   5 --
 drivers/raw/ifpga/base/opae_spi.h             |   3 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  44 +++++++++--
 drivers/raw/ifpga/ifpga_rawdev.c              |  14 ++--
 12 files changed, 205 insertions(+), 109 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 799d67d..c31a94c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -839,8 +839,13 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	struct ifpga_hw *hw;
 	u32 val;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
@@ -854,7 +859,10 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	fme->board_info.seu = id.seu;
 	fme->board_info.ptp = id.ptp;
 
-	dev_info(fme, "found: board: %s type: %s\n",
+	dev_info(fme, "found: PCI dev: %02x:%02x:%x board: %s type: %s\n",
+			hw->pci_data->bus,
+			hw->pci_data->devid,
+			hw->pci_data->function,
 			board_major_to_string(fme->board_info.major),
 			board_type_to_string(fme->board_info.type));
 
@@ -882,11 +890,11 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
-	if (max10_sys_read(MAX10_BUILD_VER, &val))
+	if (max10_sys_read(fme->max10_dev, MAX10_BUILD_VER, &val))
 		return -EINVAL;
 	fme->board_info.max10_version = val & 0xffffff;
 
-	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+	if (max10_sys_read(fme->max10_dev, NIOS2_FW_VERSION, &val))
 		return -EINVAL;
 	fme->board_info.nios_fw_version = val & 0xffffff;
 
@@ -897,12 +905,12 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	return 0;
 }
 
-static int spi_self_checking(void)
+static int spi_self_checking(struct intel_max10_device *dev)
 {
 	u32 val;
 	int ret;
 
-	ret = max10_sys_read(MAX10_TEST_REG, &val);
+	ret = max10_sys_read(dev, MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
@@ -937,10 +945,11 @@ static int fme_spi_init(struct ifpga_feature *feature)
 		goto spi_fail;
 	}
 
+
 	fme->max10_dev = max10;
 
 	/* SPI self test */
-	if (spi_self_checking()) {
+	if (spi_self_checking(max10)) {
 		ret = -EIO;
 		goto max10_fail;
 	}
@@ -1041,8 +1050,18 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 	struct ifpga_fme_hw *fme = (struct ifpga_fme_hw *)feature->parent;
 	struct altera_spi_device *spi_master;
 	struct intel_max10_device *max10;
+	struct ifpga_hw *hw;
+	struct opae_manager *mgr;
 	int ret = 0;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
+	mgr = hw->adapter->mgr;
+	if (!mgr)
+		return -ENODEV;
+
 	dev_info(fme, "FME SPI Master (NIOS) Init.\n");
 	dev_debug(fme, "FME SPI base addr %p.\n",
 			feature->addr);
@@ -1080,12 +1099,15 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 		goto release_dev;
 	}
 
+	max10->bus = hw->pci_data->bus;
+
 	fme_get_board_interface(fme);
 
 	fme->max10_dev = max10;
+	mgr->sensor_list = &max10->opae_sensor_list;
 
 	/* SPI self test */
-	if (spi_self_checking())
+	if (spi_self_checking(max10))
 		goto spi_fail;
 
 	return ret;
@@ -1344,7 +1366,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(dev, PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1372,7 +1394,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(sensor->value_reg, value)) {
+	if (max10_sys_read(dev, sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_debug.c b/drivers/raw/ifpga/base/opae_debug.c
index 88f2d5c..dad3ea3 100644
--- a/drivers/raw/ifpga/base/opae_debug.c
+++ b/drivers/raw/ifpga/base/opae_debug.c
@@ -59,6 +59,9 @@ static void opae_adapter_data_dump(void *data)
 		opae_log("OPAE Adapter Type = PCI\n");
 		opae_log("PCI Device ID: 0x%04x\n", d_pci->device_id);
 		opae_log("PCI Vendor ID: 0x%04x\n", d_pci->vendor_id);
+		opae_log("PCI bus: 0x%04x\n", d_pci->bus);
+		opae_log("PCI devid: 0x%04x\n", d_pci->devid);
+		opae_log("PCI function: 0x%04x\n", d_pci->function);
 
 		for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 			r = &d_pci->region[i];
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 1ccc967..c969dfe 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -583,11 +583,12 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_id(unsigned int id)
+opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (sensor->id == id)
 			return sensor;
 
@@ -601,11 +602,12 @@ struct opae_sensor_info *
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_name(const char *name)
+opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (!strcmp(sensor->name, name))
 			return sensor;
 
@@ -630,7 +632,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_name(name);
+	sensor = opae_mgr_get_sensor_by_name(mgr, name);
 	if (!sensor)
 		return -ENODEV;
 
@@ -658,7 +660,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_id(id);
+	sensor = opae_mgr_get_sensor_by_id(mgr, id);
 	if (!sensor)
 		return -ENODEV;
 
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index b78fbd5..cdf369f 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -40,6 +40,7 @@ struct opae_manager {
 	struct opae_adapter *adapter;
 	struct opae_manager_ops *ops;
 	struct opae_manager_networking_ops *network_ops;
+	struct opae_sensor_list *sensor_list;
 	void *data;
 };
 
@@ -75,9 +76,8 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
-extern struct opae_sensor_list opae_sensor_list;
-#define opae_mgr_for_each_sensor(sensor) \
-	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+#define opae_mgr_for_each_sensor(mgr, sensor) \
+	TAILQ_FOREACH(sensor, mgr->sensor_list, node)
 
 /* OPAE Manager APIs */
 struct opae_manager *
@@ -88,8 +88,10 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
-struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
-struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id);
 int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
 		const char *name, unsigned int *value);
 int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
@@ -241,6 +243,9 @@ struct opae_adapter_data_pci {
 	enum opae_adapter_type type;
 	u16 device_id;
 	u16 vendor_id;
+	u16 bus; /*Device bus for PCI */
+	u16 devid; /* Device ID */
+	u16 function; /* Device function */
 	struct opae_reg_region region[PCI_MAX_RESOURCE];
 	int vfio_dev_fd;  /* VFIO device file descriptor */
 };
diff --git a/drivers/raw/ifpga/base/opae_i2c.c b/drivers/raw/ifpga/base/opae_i2c.c
index f8bc247..846d751 100644
--- a/drivers/raw/ifpga/base/opae_i2c.c
+++ b/drivers/raw/ifpga/base/opae_i2c.c
@@ -28,6 +28,9 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 {
 	u8 msgbuf[2];
 	int i = 0;
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
 
 	if (flags & I2C_FLAG_ADDR16)
 		msgbuf[i++] = offset >> 8;
@@ -49,10 +52,16 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		},
 	};
 
-	if (!dev->xfer)
-		return -ENODEV;
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	ret = i2c_transfer(dev, msg, 2);
 
-	return i2c_transfer(dev, msg, 2);
+exit:
+	pthread_mutex_unlock(&dev->lock);
+	return ret;
 }
 
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
@@ -63,12 +72,18 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	int ret;
 	int i = 0;
 
-	if (!dev->xfer)
-		return -ENODEV;
+	pthread_mutex_lock(&dev->lock);
+
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
 
 	buf = opae_malloc(I2C_MAX_OFFSET_LEN + len);
-	if (!buf)
-		return -ENOMEM;
+	if (!buf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
 
 	msg.addr = slave_addr;
 	msg.flags = 0;
@@ -84,6 +99,8 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	ret = i2c_transfer(dev, &msg, 1);
 
 	opae_free(buf);
+exit:
+	pthread_mutex_unlock(&dev->lock);
 	return ret;
 }
 
@@ -477,14 +494,19 @@ struct altera_i2c_dev *altera_i2c_probe(void *base)
 	dev->i2c_clk = dev->i2c_param.ref_clk * 1000000;
 	dev->xfer = altera_i2c_xfer;
 
+	if (pthread_mutex_init(&dev->lock, NULL))
+		return NULL;
+
 	altera_i2c_hardware_init(dev);
 
 	return dev;
 }
 
-int altera_i2c_remove(struct altera_i2c_dev *dev)
+void altera_i2c_remove(struct altera_i2c_dev *dev)
 {
-	altera_i2c_disable(dev);
-
-	return 0;
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
+		altera_i2c_disable(dev);
+		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/base/opae_i2c.h b/drivers/raw/ifpga/base/opae_i2c.h
index 8890c8f..266e127 100644
--- a/drivers/raw/ifpga/base/opae_i2c.h
+++ b/drivers/raw/ifpga/base/opae_i2c.h
@@ -93,6 +93,7 @@ struct altera_i2c_dev {
 	u32 isr_mask;
 	u8 *buf;
 	int (*xfer)(struct altera_i2c_dev *dev, struct i2c_msg *msg, int num);
+	pthread_mutex_t lock;
 };
 
 /**
@@ -114,7 +115,7 @@ enum i2c_msg_flags {
 };
 
 struct altera_i2c_dev *altera_i2c_probe(void *base);
-int altera_i2c_remove(struct altera_i2c_dev *dev);
+void altera_i2c_remove(struct altera_i2c_dev *dev);
 int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		u32 offset, u8 *buf, u32 count);
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 748ab56..8e23ca1 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -5,45 +5,50 @@
 #include "opae_intel_max10.h"
 #include <libfdt.h>
 
-static struct intel_max10_device *g_max10;
-
-struct opae_sensor_list opae_sensor_list =
-	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
-
-int max10_reg_read(unsigned int reg, unsigned int *val)
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_read(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x\n", __func__, dev->bus, reg);
+
+	return spi_transaction_read(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)val);
 }
 
-int max10_reg_write(unsigned int reg, unsigned int val)
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val)
 {
 	unsigned int tmp = val;
 
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_write(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x, val:0x%x\n", __func__,
+			dev->bus, reg, val);
+
+	return spi_transaction_write(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)&tmp);
 }
 
-int max10_sys_read(unsigned int offset, unsigned int *val)
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_read(g_max10->base + offset, val);
+
+	return max10_reg_read(dev, dev->base + offset, val);
 }
 
-int max10_sys_write(unsigned int offset, unsigned int val)
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_write(g_max10->base + offset, val);
+	return max10_reg_write(dev, dev->base + offset, val);
 }
 
 static struct max10_compatible_id max10_id_table[] = {
@@ -86,8 +91,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
-static int altera_nor_flash_read(u32 offset,
-		void *buffer, u32 len)
+static int altera_nor_flash_read(struct intel_max10_device *dev,
+	u32 offset, void *buffer, u32 len)
 {
 	int word_len;
 	int i;
@@ -95,13 +100,13 @@ static int altera_nor_flash_read(u32 offset,
 	unsigned int value;
 	int ret;
 
-	if (!buffer || len <= 0)
+	if (!dev || !buffer || len <= 0)
 		return -ENODEV;
 
 	word_len = len/4;
 
 	for (i = 0; i < word_len; i++) {
-		ret = max10_reg_read(offset + i*4,
+		ret = max10_reg_read(dev, offset + i*4,
 				&value);
 		if (ret)
 			return -EBUSY;
@@ -112,12 +117,12 @@ static int altera_nor_flash_read(u32 offset,
 	return 0;
 }
 
-static int enable_nor_flash(bool on)
+static int enable_nor_flash(struct intel_max10_device *dev, bool on)
 {
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_sys_read(RSU_REG, &val);
+	ret = max10_sys_read(dev, RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -128,7 +133,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_sys_write(RSU_REG, val);
+	return max10_sys_write(dev, RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -140,7 +145,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_sys_read(DT_AVAIL_REG, &val);
+	ret = max10_sys_read(max10, DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -151,19 +156,19 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
+	ret = max10_sys_read(max10, DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
 	}
 
-	ret = enable_nor_flash(true);
+	ret = enable_nor_flash(max10, true);
 	if (ret) {
 		dev_err(max10 "fail to enable flash\n");
 		return ret;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	ret = altera_nor_flash_read(max10, dt_addr, &hdr, sizeof(hdr));
 	if (ret) {
 		dev_err(max10 "read fdt header fail\n");
 		goto done;
@@ -188,7 +193,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		goto done;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	ret = altera_nor_flash_read(max10, dt_addr, fdt_root, dt_size);
 	if (ret) {
 		dev_err(max10 "cannot read device table\n");
 		goto done;
@@ -207,7 +212,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	max10->fdt_root = fdt_root;
 
 done:
-	ret = enable_nor_flash(false);
+	ret = enable_nor_flash(max10, false);
 
 	if (ret && fdt_root)
 		opae_free(fdt_root);
@@ -298,12 +303,12 @@ static int fdt_get_named_reg(const void *fdt, int node, const char *name,
 	return fdt_get_reg(fdt, node, idx, start, size);
 }
 
-static void max10_sensor_uinit(void)
+static void max10_sensor_uinit(struct intel_max10_device *dev)
 {
 	struct opae_sensor_info *info;
 
-	TAILQ_FOREACH(info, &opae_sensor_list, node) {
-		TAILQ_REMOVE(&opae_sensor_list, info, node);
+	TAILQ_FOREACH(info, &dev->opae_sensor_list, node) {
+		TAILQ_REMOVE(&dev->opae_sensor_list, info, node);
 		opae_free(info);
 	}
 }
@@ -313,8 +318,8 @@ static bool sensor_reg_valid(struct sensor_reg *reg)
 	return !!reg->size;
 }
 
-static int max10_add_sensor(struct raw_sensor_info *info,
-		struct opae_sensor_info *sensor)
+static int max10_add_sensor(struct intel_max10_device *dev,
+	struct raw_sensor_info *info, struct opae_sensor_info *sensor)
 {
 	int i;
 	int ret = 0;
@@ -332,12 +337,15 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_sys_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(dev, info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
-		if (val == 0xdeadbeef)
+		if (val == 0xdeadbeef) {
+			dev_debug(dev, "%s: sensor:%s invalid 0x%x at:%d\n",
+				__func__, sensor->name, val, i);
 			continue;
+		}
 
 		val *= info->multiplier;
 
@@ -458,7 +466,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
 		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
 
-		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+		dev_debug(dev, "found sensor from DTB: %s: %s: %u: %u\n",
 				raw->name, raw->type,
 				raw->id, raw->multiplier);
 
@@ -473,15 +481,16 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 			goto free_sensor;
 		}
 
-		if (max10_add_sensor(raw, sensor)) {
+		if (max10_add_sensor(dev, raw, sensor)) {
 			ret = -EINVAL;
 			opae_free(sensor);
 			goto free_sensor;
 		}
 
-		if (sensor->flags & OPAE_SENSOR_VALID)
-			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
-		else
+		if (sensor->flags & OPAE_SENSOR_VALID) {
+			TAILQ_INSERT_TAIL(&dev->opae_sensor_list, sensor, node);
+			dev_info(dev, "found valid sensor: %s\n", sensor->name);
+		} else
 			opae_free(sensor);
 
 		opae_free(raw);
@@ -492,7 +501,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 free_sensor:
 	if (raw)
 		opae_free(raw);
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 	return ret;
 }
 
@@ -500,7 +509,7 @@ static int check_max10_version(struct intel_max10_device *dev)
 {
 	unsigned int v;
 
-	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+	if (!max10_reg_read(dev, MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
 				&v)) {
 		if (v != 0xffffffff) {
 			dev_info(dev, "secure MAX10 detected\n");
@@ -565,6 +574,8 @@ struct intel_max10_device *
 	if (!dev)
 		return NULL;
 
+	TAILQ_INIT(&dev->opae_sensor_list);
+
 	dev->spi_master = spi;
 
 	dev->spi_tran_dev = spi_transaction_init(spi, chipselect);
@@ -573,9 +584,6 @@ struct intel_max10_device *
 		goto free_dev;
 	}
 
-	/* set the max10 device firstly */
-	g_max10 = dev;
-
 	/* check the max10 version */
 	ret = check_max10_version(dev);
 	if (ret) {
@@ -601,7 +609,7 @@ struct intel_max10_device *
 	}
 
 	/* read FPGA loading information */
-	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
+	ret = max10_sys_read(dev, FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
 		goto release_max10_hw;
@@ -611,14 +619,13 @@ struct intel_max10_device *
 	return dev;
 
 release_max10_hw:
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return NULL;
@@ -629,7 +636,7 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
@@ -637,7 +644,6 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index e632941..123cdc4 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -26,6 +26,9 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SECURE		BIT(6)
 #define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
@@ -33,6 +36,8 @@ struct intel_max10_device {
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
 	unsigned int base; /* max10 base address */
+	u16 bus;
+	struct opae_sensor_list opae_sensor_list;
 };
 
 /* retimer speed */
@@ -136,17 +141,19 @@ struct opae_retimer_status {
 
 #define DFT_MAX_SIZE		0x7e0000
 
-int max10_reg_read(unsigned int reg, unsigned int *val);
-int max10_reg_write(unsigned int reg, unsigned int val);
-int max10_sys_read(unsigned int offset, unsigned int *val);
-int max10_sys_write(unsigned int offset, unsigned int val);
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val);
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val);
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val);
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
-/** List of opae sensors */
-TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
 
 #define SENSOR_REG_VALUE 0x0
 #define SENSOR_REG_HIGH_WARN 0x1
diff --git a/drivers/raw/ifpga/base/opae_spi.c b/drivers/raw/ifpga/base/opae_spi.c
index cc52782..bfdc83e 100644
--- a/drivers/raw/ifpga/base/opae_spi.c
+++ b/drivers/raw/ifpga/base/opae_spi.c
@@ -181,7 +181,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 	u32 rxd;
 	unsigned int tx_data;
 	u32 status;
-	int retry = 0;
 	int ret;
 
 	while (count < dev->len) {
@@ -194,10 +193,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 				return -EIO;
 			if (status & ALTERA_SPI_STATUS_RRDY_MSK)
 				break;
-			if (retry++ > SPI_MAX_RETRY) {
-				dev_err(dev, "%s, read timeout\n", __func__);
-				return -EBUSY;
-			}
 		}
 
 		ret = spi_reg_read(dev, ALTERA_SPI_RXDATA, &rxd);
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index 6355deb..d20a4c3 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -38,7 +38,7 @@
 #define SPI_WRITE 0x20
 #define WRITE_DATA_MASK GENMASK_ULL(31, 0)
 
-#define SPI_MAX_RETRY 100000
+#define SPI_MAX_RETRY 1000000
 
 #define TYPE_SPI 0
 #define TYPE_NIOS_SPI 1
@@ -102,6 +102,7 @@ struct spi_transaction_dev {
 	struct altera_spi_device *dev;
 	int chipselect;
 	struct spi_tran_buffer *buffer;
+	pthread_mutex_t lock;
 };
 
 struct spi_tran_header {
diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 06ca625..013efee 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -154,6 +154,8 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	unsigned int rx_len = 0;
 	int retry = 0;
 	int spi_flags;
+	unsigned long timeout = msecs_to_timer_cycles(1000);
+	unsigned long ticks;
 	unsigned int resp_max_len = 2 * resp_len;
 
 	print_buffer("before bytes:", send_data, send_len);
@@ -207,8 +209,11 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	if (ret != SPI_FOUND_EOP) {
 		tx_buffer = NULL;
 		tx_len = 0;
-		if (retry++ > 10) {
-			dev_err(NULL, "cannot found valid data from SPI\n");
+		ticks = rte_get_timer_cycles();
+		if (time_after(ticks, timeout) &&
+				retry++ > SPI_MAX_RETRY) {
+			dev_err(NULL, "Have retry %d, found invalid packet data\n",
+				retry);
 			return -EBUSY;
 		}
 
@@ -427,23 +432,36 @@ static int do_transaction(struct spi_transaction_dev *dev, unsigned int addr,
 int spi_transaction_read(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_READ : SPI_TRAN_NON_SEQ_READ);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 int spi_transaction_write(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_WRITE : SPI_TRAN_NON_SEQ_WRITE);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 		int chipselect)
 {
 	struct spi_transaction_dev *spi_tran_dev;
+	int ret;
 
 	spi_tran_dev = opae_malloc(sizeof(struct spi_transaction_dev));
 	if (!spi_tran_dev)
@@ -453,18 +471,28 @@ struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 	spi_tran_dev->chipselect = chipselect;
 
 	spi_tran_dev->buffer = opae_malloc(sizeof(struct spi_tran_buffer));
-	if (!spi_tran_dev->buffer) {
-		opae_free(spi_tran_dev);
-		return NULL;
+	if (!spi_tran_dev->buffer)
+		goto err;
+
+	ret = pthread_mutex_init(&spi_tran_dev->lock, NULL);
+	if (ret) {
+		dev_err(spi_tran_dev, "fail to init mutex lock\n");
+		goto err;
 	}
 
 	return spi_tran_dev;
+
+err:
+	opae_free(spi_tran_dev);
+	return NULL;
 }
 
 void spi_transaction_remove(struct spi_transaction_dev *dev)
 {
 	if (dev && dev->buffer)
 		opae_free(dev->buffer);
-	if (dev)
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
 		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 5701e2d..88d632d 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -361,7 +361,7 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 	if (!mgr)
 		return -ENODEV;
 
-	opae_mgr_for_each_sensor(sensor) {
+	opae_mgr_for_each_sensor(mgr, sensor) {
 		if (!(sensor->flags & OPAE_SENSOR_VALID))
 			goto fail;
 
@@ -370,8 +370,8 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 			goto fail;
 
 		if (value == 0xdeadbeef) {
-			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
-					sensor->name, value);
+			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value %x\n",
+					raw_dev->dev_id, sensor->name, value);
 			continue;
 		}
 
@@ -394,8 +394,9 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 		/* monitor 12V AUX sensor */
 		if (!strcmp(sensor->name, "12V AUX Voltage")) {
 			if (value < AUX_VOLTAGE_WARN) {
-				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
-						sensor->name, value);
+				IFPGA_RAWDEV_PMD_INFO(
+					"%s reach theshold %d mV\n",
+					sensor->name, value);
 				*gsd_start = true;
 				break;
 			}
@@ -1437,6 +1438,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->bus = pci_dev->addr.bus;
+	data->devid = pci_dev->addr.devid;
+	data->function = pci_dev->addr.function;
 	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
-- 
1.8.3.1

[dpdk-dev] [PATCH v15 19/19] raw/ifpga: introducing new irq API

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing new register and unregister API for ifpga interrupt.
1. register FME and AFU interrupt
   ifpga_register_msix_irq()

2. unregister FME and AFU interrupt
   ifpga_unregister_msix_irq()

On PAC N3000 card, there is one PCIe MSIX interrupt for FME managerment,
like the error report, thermal management, we use this interrupt in
ifpga_rawdev device driver. on the other hand, there are about 4 PCIe
MSIX interrupts are reserved for AFU which end-user can use those
interrupts in their AFU logic design. End-user can use those APIs to
register interrupt handler in their AFU drivers.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 105 +++++++++++++++++++++++++++------------
 drivers/raw/ifpga/ifpga_rawdev.h |  14 ++++++
 2 files changed, 88 insertions(+), 31 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 88d632d..c930710 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -79,6 +79,10 @@
 static int ifpga_monitor_start;
 static pthread_t ifpga_monitor_start_thread;
 
+#define IFPGA_MAX_IRQ 12
+/* 0 for FME interrupt, others are reserved for AFU irq */
+static struct rte_intr_handle ifpga_irq_handle[IFPGA_MAX_IRQ];
+
 static struct ifpga_rawdev *
 ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
 static int set_surprise_link_check_aer(
@@ -1332,53 +1336,90 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 	fme_err_handle_catfatal_error(mgr);
 }
 
-static struct rte_intr_handle fme_intr_handle;
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg)
+{
+	struct rte_intr_handle intr_handle;
+
+	if (type == IFPGA_FME_IRQ)
+		intr_handle = ifpga_irq_handle[0];
+	else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+	rte_intr_efd_disable(&intr_handle);
+
+	return rte_intr_callback_unregister(&intr_handle,
+			handler, arg);
+}
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg)
 {
 	int ret;
-	struct fpga_fme_err_irq_set err_irq_set;
+	struct rte_intr_handle intr_handle;
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_accelerator *acc;
 
-	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+	adapter = ifpga_rawdev_get_priv(dev);
+	if (!adapter)
+		return -ENODEV;
 
-	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
-	if (ret)
-		return -EINVAL;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
 
-	fme_intr_handle.fd = fme_intr_handle.efds[0];
+	if (type == IFPGA_FME_IRQ) {
+		intr_handle = ifpga_irq_handle[0];
+		count = 1;
+	} else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
-			fme_intr_handle.vfio_dev_fd,
-			fme_intr_handle.efds[0], fme_intr_handle.fd);
+	intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
 
-	err_irq_set.evtfd = fme_intr_handle.efds[0];
-	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	ret = rte_intr_efd_enable(&intr_handle, count);
 	if (ret)
-		return -EINVAL;
+		return -ENODEV;
+
+	intr_handle.fd = intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("register %s irq, vfio_fd=%d, fd=%d\n",
+			name, intr_handle.vfio_dev_fd,
+			intr_handle.fd);
+
+	if (type == IFPGA_FME_IRQ) {
+		struct fpga_fme_err_irq_set err_irq_set;
+		err_irq_set.evtfd = intr_handle.efds[0];
+
+		ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+		if (ret)
+			return -EINVAL;
+	} else if (type == IFPGA_AFU_IRQ) {
+		acc = opae_adapter_get_acc(adapter, port_id);
+		if (!acc)
+			return -EINVAL;
 
-	/* register FME interrupt using DPDK API */
-	ret = rte_intr_callback_register(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
+		ret = opae_acc_set_irq(acc, vec_start, count, intr_handle.efds);
+		if (ret)
+			return -EINVAL;
+	}
+
+	/* register interrupt handler using DPDK API */
+	ret = rte_intr_callback_register(&intr_handle,
+			handler, (void *)arg);
 	if (ret)
 		return -EINVAL;
 
-	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+	IFPGA_RAWDEV_PMD_INFO("success register %s interrupt\n", name);
 
 	return 0;
 }
 
 static int
-ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
-{
-	rte_intr_efd_disable(&fme_intr_handle);
-
-	return rte_intr_callback_unregister(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
-}
-
-static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -1467,7 +1508,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
-	ret = ifpga_register_fme_interrupt(mgr);
+	ret = ifpga_register_msix_irq(rawdev, 0, IFPGA_FME_IRQ, 0, 0,
+			fme_interrupt_handler, "fme_irq", mgr);
 	if (ret)
 		goto free_adapter_data;
 
@@ -1519,7 +1561,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	if (!mgr)
 		return -ENODEV;
 
-	if (ifpga_unregister_fme_interrupt(mgr))
+	if (ifpga_unregister_msix_irq(IFPGA_FME_IRQ, 0,
+				fme_interrupt_handler, mgr))
 		return -EINVAL;
 
 	opae_adapter_data_free(adapter->data);
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index bd42083..7754beb 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -62,4 +62,18 @@ struct ifpga_rawdev {
 struct ifpga_rawdev *
 ifpga_rawdev_get(const struct rte_rawdev *rawdev);
 
+enum ifpga_irq_type {
+	IFPGA_FME_IRQ = 0,
+	IFPGA_AFU_IRQ = 1,
+};
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg);
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Rosen Xu]

This patch set adds PCIe AER disable and FPGA interrupt support for
ipn3ke. It also provides a small rework for port bonding between FPGA
line side port and I40e PF port.

What is the PCI Express AER(Advanced Error Reporting)?
Advanced Error Reporting capability is implemented with a PCI Express
advanced error reporting extended capability structure providing more
robust error reporting. It's also one of PCI Express error reporting
paradigms. AER is supported by most of PCIe devices.

In PAC N3000 card, some uncertainty errors will cause FPGA reload,
such as temperature is higher than threshold. From Software point of
view, FPGA reload means FPGA unplug and plug. For avoiding system
crash we need to clear AER register before these errors occur.

Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga
rawdev driver, we implement a FME interrupt function to notify errors
reported by FME. Besides this, OPAE share code also provide a common
AFU interrupt API for users to register their own interrupt functions.

v16 updates:
=========
- rebase to DPDK 19.11 RC-2.

v15 updates:
=========
- fix share library build issues.

v14 updates:
=========
- fix coding style and remove unnecessary comments.

v13 updates:
=========
- fix meson build issue.

v12 updates:
=========
- fix meson build issue.
 
v11 updates:
=========
- move symbol ifpga_rawdev_ge to the EXPERIMENTAL section of 
  drivers/raw/ifpga/rte_rawdev_ifpga_version.map

v10 updates:
=========
- introducing new irq API
- fix meson build issue

v9 updates:
=========
- Add mutex lock on do_transaction() function for SPI driver to avoid race
  condition.

v8 updates:
=========
- add multiple cards support.

v7 updates:
==========
- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
- function rte_pmd_i40e_set_switch_dev works as an external API,
  use port_id but not rte_eth_dev as parameter.
- add doxygen header here for the new API.
- update the rte_pmd_i40e_version.map.
- fix coding style issue.
- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
- for functions with a lot of similarity, extract out common function to reduce
  duplication.

v6 updates:
=========
- correct author information.
- correct typo in commit message and remove Gerrit Change-Id's before
  submitting upstream

v5 updates:
==========
- add lightweight fpga image support. in lightweight fpga image mode,
  ipn3ke representor will not be probed.

v4 updates:
===========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support

Andy Pei (2):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga: add lightweight fpga image support

Rosen Xu (3):
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei zhang (14):
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga/base: add secure support
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors
  raw/ifpga/base: add new API get board info
  raw/ifpga/base: add multiple cards support
  raw/ifpga: introducing new irq API

 config/common_base                            |   4 +-
 config/common_linux                           |   6 +
 drivers/meson.build                           |   6 +-
 drivers/net/i40e/base/i40e_type.h             |   3 +
 drivers/net/i40e/i40e_ethdev.c                |   6 +
 drivers/net/i40e/rte_pmd_i40e.c               |  21 +
 drivers/net/i40e/rte_pmd_i40e.h               |  18 +
 drivers/net/i40e/rte_pmd_i40e_version.map     |   8 +-
 drivers/net/ipn3ke/Makefile                   |   1 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 292 ++-------
 drivers/net/ipn3ke/ipn3ke_flow.c              |   6 +
 drivers/net/ipn3ke/ipn3ke_rawdev_api.h        |  12 +
 drivers/net/ipn3ke/ipn3ke_representor.c       |  10 +-
 drivers/net/ipn3ke/meson.build                |  24 +-
 drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map |   6 +
 drivers/raw/ifpga/Makefile                    |   5 +
 drivers/raw/ifpga/base/ifpga_api.c            |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h        |  75 ++-
 drivers/raw/ifpga/base/ifpga_feature_dev.c    |  59 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c            | 166 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c      |  74 ++-
 drivers/raw/ifpga/base/ifpga_hw.h             |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c           |  18 +
 drivers/raw/ifpga/base/ifpga_port_error.c     |  19 +
 drivers/raw/ifpga/base/meson.build            |   2 +-
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          | 137 ++++
 drivers/raw/ifpga/base/opae_hw_api.h          |  26 +
 drivers/raw/ifpga/base/opae_i2c.c             |  44 +-
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c     | 599 ++++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h     | 157 ++++-
 drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
 drivers/raw/ifpga/base/opae_spi.c             |   5 -
 drivers/raw/ifpga/base/opae_spi.h             |  26 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  84 ++-
 drivers/raw/ifpga/ifpga_rawdev.c              | 907 +++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h              |  30 +
 drivers/raw/ifpga/meson.build                 |  26 +-
 mk/rte.app.mk                                 |   2 +-
 42 files changed, 2484 insertions(+), 441 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v16 01/19] net/i40e: i40e support ipn3ke FPGA port bonding

[Rosen Xu]

From: Andy Pei <andy.pei@intel.com>

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h         |  3 +++
 drivers/net/i40e/i40e_ethdev.c            |  6 ++++++
 drivers/net/i40e/rte_pmd_i40e.c           | 21 +++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h           | 18 ++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e_version.map |  8 +++++++-
 5 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 9a81f99..1afddeb 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1393,6 +1393,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2906,6 +2909,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index 4c3c708..fdcb1a4 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -3207,3 +3207,24 @@ int rte_pmd_i40e_flow_add_del_packet_template(
 	I40E_WRITE_FLUSH(hw);
 	return 0;
 }
+
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev)
+{
+	struct rte_eth_dev *i40e_dev;
+	struct i40e_hw *hw;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+	i40e_dev = &rte_eth_devices[port_id];
+	if (!is_i40e_supported(i40e_dev))
+		return -ENOTSUP;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+	if (!hw)
+		return -1;
+
+	hw->switch_dev = switch_dev;
+
+	return 0;
+}
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..355b8be 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+/**
+ * For ipn3ke, i40e works with FPGA.
+ * In this situation, i40e get link status from fpga,
+ * fpga works as switch_dev for i40e.
+ * This function set switch_dev for i40e.
+ *
+ * @param inset
+ *    Input set value.
+ * @param field_idx
+ *    Field index for input set.
+ * @return
+ *   - (less than 0) if failed.
+ *   - (0) if success.
+ */
+__rte_experimental
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
index cccd576..79641f2 100644
--- a/drivers/net/i40e/rte_pmd_i40e_version.map
+++ b/drivers/net/i40e/rte_pmd_i40e_version.map
@@ -64,4 +64,10 @@ DPDK_18.02 {
 
 	rte_pmd_i40e_inset_get;
 	rte_pmd_i40e_inset_set;
-} DPDK_17.11;
\ No newline at end of file
+} DPDK_17.11;
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_i40e_set_switch_dev;
+} DPDK_18.02;
\ No newline at end of file
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v16 01/19] net/i40e: i40e support ipn3ke FPGA port bonding

[Ferruh Yigit]

On 11/13/2019 7:08 AM, Rosen Xu wrote:
> From: Andy Pei <andy.pei@intel.com>
> 
> In ipn3ke, each FPGA network side port bonding to an i40e pf,
> each i40e pf link status should get data from FPGA network,
> side port. This patch provide bonding relationship.
> 
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>

<...>

> @@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
>  	return 0;
>  }
>  
> +/**
> + * For ipn3ke, i40e works with FPGA.
> + * In this situation, i40e get link status from fpga,
> + * fpga works as switch_dev for i40e.
> + * This function set switch_dev for i40e.
> + *
> + * @param inset
> + *    Input set value.
> + * @param field_idx
> + *    Field index for input set.

These params are wrong, can you please provide proper information, I can update
them in next-net?

> + * @return
> + *   - (less than 0) if failed.
> + *   - (0) if success.
> + */
> +__rte_experimental
> +int
> +rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
> +
>  #endif /* _PMD_I40E_H_ */
> diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
> index cccd576..79641f2 100644
> --- a/drivers/net/i40e/rte_pmd_i40e_version.map
> +++ b/drivers/net/i40e/rte_pmd_i40e_version.map
> @@ -64,4 +64,10 @@ DPDK_18.02 {
>  
>  	rte_pmd_i40e_inset_get;
>  	rte_pmd_i40e_inset_set;
> -} DPDK_17.11;
> \ No newline at end of file
> +} DPDK_17.11;
> +
> +EXPERIMENTAL {
> +	global:
> +
> +	rte_pmd_i40e_set_switch_dev;
> +} DPDK_18.02;

The 'DPDK_18.02' part is not required for experimental block, I will remove it
while merging.

Re: [dpdk-dev] [PATCH v16 01/19] net/i40e: i40e support ipn3ke FPGA port bonding

[Ferruh Yigit]

On 11/13/2019 2:38 PM, Ferruh Yigit wrote:
> On 11/13/2019 7:08 AM, Rosen Xu wrote:
>> From: Andy Pei <andy.pei@intel.com>
>>
>> In ipn3ke, each FPGA network side port bonding to an i40e pf,
>> each i40e pf link status should get data from FPGA network,
>> side port. This patch provide bonding relationship.
>>
>> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
>> Signed-off-by: Andy Pei <andy.pei@intel.com>
> 
> <...>
> 
>> @@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
>>  	return 0;
>>  }
>>  
>> +/**
>> + * For ipn3ke, i40e works with FPGA.
>> + * In this situation, i40e get link status from fpga,
>> + * fpga works as switch_dev for i40e.
>> + * This function set switch_dev for i40e.
>> + *
>> + * @param inset
>> + *    Input set value.
>> + * @param field_idx
>> + *    Field index for input set.
> 
> These params are wrong, can you please provide proper information, I can update
> them in next-net?
> 
>> + * @return
>> + *   - (less than 0) if failed.
>> + *   - (0) if success.
>> + */
>> +__rte_experimental
>> +int
>> +rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
>> +
>>  #endif /* _PMD_I40E_H_ */
>> diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
>> index cccd576..79641f2 100644
>> --- a/drivers/net/i40e/rte_pmd_i40e_version.map
>> +++ b/drivers/net/i40e/rte_pmd_i40e_version.map
>> @@ -64,4 +64,10 @@ DPDK_18.02 {
>>  
>>  	rte_pmd_i40e_inset_get;
>>  	rte_pmd_i40e_inset_set;
>> -} DPDK_17.11;
>> \ No newline at end of file
>> +} DPDK_17.11;
>> +
>> +EXPERIMENTAL {
>> +	global:
>> +
>> +	rte_pmd_i40e_set_switch_dev;
>> +} DPDK_18.02;
> 
> The 'DPDK_18.02' part is not required for experimental block, I will remove it
> while merging.
> 

Since there will be a new version, can you please fix these too in next version?

Re: [dpdk-dev] [PATCH v16 01/19] net/i40e: i40e support ipn3ke FPGA port bonding

[Xu, Rosen]

> -----Original Message-----
> From: Yigit, Ferruh
> Sent: Wednesday, November 13, 2019 22:38
> To: Xu, Rosen <rosen.xu@intel.com>; dev@dpdk.org
> Cc: Zhang, Tianfei <tianfei.zhang@intel.com>; Pei, Andy
> <andy.pei@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>
> Subject: Re: [PATCH v16 01/19] net/i40e: i40e support ipn3ke FPGA port
> bonding
> 
> On 11/13/2019 7:08 AM, Rosen Xu wrote:
> > From: Andy Pei <andy.pei@intel.com>
> >
> > In ipn3ke, each FPGA network side port bonding to an i40e pf, each
> > i40e pf link status should get data from FPGA network, side port. This
> > patch provide bonding relationship.
> >
> > Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> 
> <...>
> 
> > @@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port,
> uint8_t pctype,
> >  	return 0;
> >  }
> >
> > +/**
> > + * For ipn3ke, i40e works with FPGA.
> > + * In this situation, i40e get link status from fpga,
> > + * fpga works as switch_dev for i40e.
> > + * This function set switch_dev for i40e.
> > + *
> > + * @param inset
> > + *    Input set value.
> > + * @param field_idx
> > + *    Field index for input set.
> 
> These params are wrong, can you please provide proper information, I can
> update them in next-net?

Fixed in v17.

> > + * @return
> > + *   - (less than 0) if failed.
> > + *   - (0) if success.
> > + */
> > +__rte_experimental
> > +int
> > +rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev
> > +*switch_dev);
> > +
> >  #endif /* _PMD_I40E_H_ */
> > diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map
> > b/drivers/net/i40e/rte_pmd_i40e_version.map
> > index cccd576..79641f2 100644
> > --- a/drivers/net/i40e/rte_pmd_i40e_version.map
> > +++ b/drivers/net/i40e/rte_pmd_i40e_version.map
> > @@ -64,4 +64,10 @@ DPDK_18.02 {
> >
> >  	rte_pmd_i40e_inset_get;
> >  	rte_pmd_i40e_inset_set;
> > -} DPDK_17.11;
> > \ No newline at end of file
> > +} DPDK_17.11;
> > +
> > +EXPERIMENTAL {
> > +	global:
> > +
> > +	rte_pmd_i40e_set_switch_dev;
> > +} DPDK_18.02;
> 
> The 'DPDK_18.02' part is not required for experimental block, I will remove it
> while merging.

Fixed in v17.

[dpdk-dev] [PATCH v16 02/19] raw/ifpga/base: add irq support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME global error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

To build this feature, CONFIG_RTE_EAL_VFIO should be enabled.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                         |  2 +-
 config/common_linux                        |  6 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 59 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 19 ++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 18 +++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 19 ++++++++++
 6 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/config/common_base b/config/common_base
index 9142778..36ab13d 100644
--- a/config/common_base
+++ b/config/common_base
@@ -788,7 +788,7 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
 #
 # Compile PMD for Intel FPGA raw device
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
 
 #
 # Compile PMD for Intel IOAT raw device
diff --git a/config/common_linux b/config/common_linux
index 96e2e1f..a78b8c6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -68,3 +68,9 @@ CONFIG_RTE_LIBRTE_HINIC_PMD=y
 # Hisilicon HNS3 PMD driver
 #
 CONFIG_RTE_LIBRTE_HNS3_PMD=y
+
+#
+# Compile PMD for Intel FPGA raw device
+# To compile, CONFIG_RTE_EAL_VFIO should be enabled.
+#
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..0f852a7 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,61 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+#define FPGA_MAX_MSIX_VEC_COUNT	128
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MAX_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	/* 0 < irq_set->count < FPGA_MAX_MSIX_VEC_COUNT */
+	irq_set->count = count ?
+		(count > FPGA_MAX_MSIX_VEC_COUNT ?
+		 FPGA_MAX_MSIX_VEC_COUNT : count) : 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..2978c79 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,28 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&fme->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..c0aaf01 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,27 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..189f762 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,28 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 03/19] raw/ifpga/base: clear pending bit

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 2978c79..be041ec 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 04/19] raw/ifpga/base: add SEU error support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 31 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index be041ec..5d6d630 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,33 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr(struct ifpga_fme_hw *fme,
+		u64 *val, bool high)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+				FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	if (high)
+		*val = readq(&fme_err->seu_emr_h);
+	else
+		*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +297,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr(fme, &prop->data, 0);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr(fme, &prop->data, 1);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 05/19] raw/ifpga/base: add device tree support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

To eliminate build error, we currently do not compile raw/ifpga
and net/ipn3ke. User should install libfdt and libfdt-devel first,
modify config/common_linux, CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
to CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y, modify config/common_base,
CONFIG_RTE_LIBRTE_IPN3KE_PMD=n to CONFIG_RTE_LIBRTE_IPN3KE_PMD=y.
Then this function can work.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 config/common_base                        |   2 +-
 config/common_linux                       |   2 +-
 drivers/meson.build                       |   6 +-
 drivers/net/ipn3ke/meson.build            |  24 +++-
 drivers/raw/ifpga/base/meson.build        |   2 +-
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 drivers/raw/ifpga/meson.build             |  25 ++--
 mk/rte.app.mk                             |   2 +-
 9 files changed, 235 insertions(+), 21 deletions(-)

diff --git a/config/common_base b/config/common_base
index 36ab13d..7c015f9 100644
--- a/config/common_base
+++ b/config/common_base
@@ -347,7 +347,7 @@ CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
 #
 # Compile burst-oriented IPN3KE PMD driver
 #
-CONFIG_RTE_LIBRTE_IPN3KE_PMD=y
+CONFIG_RTE_LIBRTE_IPN3KE_PMD=n
 
 #
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
diff --git a/config/common_linux b/config/common_linux
index a78b8c6..c5cf3d6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -73,4 +73,4 @@ CONFIG_RTE_LIBRTE_HNS3_PMD=y
 # Compile PMD for Intel FPGA raw device
 # To compile, CONFIG_RTE_EAL_VFIO should be enabled.
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
diff --git a/drivers/meson.build b/drivers/meson.build
index 823e3ef..b7fdfb7 100644
--- a/drivers/meson.build
+++ b/drivers/meson.build
@@ -9,12 +9,12 @@ endif
 dpdk_driver_classes = ['common',
 	       'bus',
 	       'mempool', # depends on common and bus.
-	       'net',     # depends on common, bus and mempool.
+	       'raw',     # depends on common and bus.
+	       'net',     # depends on common, bus, mempool and raw.
 	       'crypto',  # depends on common, bus and mempool (net in future).
 	       'compress', # depends on common, bus, mempool.
 	       'event',   # depends on common, bus, mempool and net.
-	       'baseband', # depends on common and bus.
-	       'raw']     # depends on common, bus, mempool, net and event.
+	       'baseband'] # depends on common and bus.
 
 disabled_drivers = get_option('disable_drivers').split(',')
 
diff --git a/drivers/net/ipn3ke/meson.build b/drivers/net/ipn3ke/meson.build
index 74b4d7c..e3c8a67 100644
--- a/drivers/net/ipn3ke/meson.build
+++ b/drivers/net/ipn3ke/meson.build
@@ -8,10 +8,22 @@
 #  rte_eth_dev_destroy()
 #  rte_eth_switch_domain_free()
 #
-allow_experimental_apis = true
 
-sources += files('ipn3ke_ethdev.c',
-	'ipn3ke_representor.c',
-	'ipn3ke_tm.c',
-	'ipn3ke_flow.c')
-deps += ['bus_ifpga', 'sched']
+dep = dependency('libfdt', required: false)
+if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
+	build = false
+	reason = 'missing dependency, "libfdt"'
+endif
+
+if build
+	allow_experimental_apis = true
+
+	sources += files('ipn3ke_ethdev.c',
+		'ipn3ke_representor.c',
+		'ipn3ke_tm.c',
+		'ipn3ke_flow.c')
+	deps += ['bus_ifpga', 'sched', 'pmd_i40e', 'rawdev', 'rawdev_ifpga']
+endif
diff --git a/drivers/raw/ifpga/base/meson.build b/drivers/raw/ifpga/base/meson.build
index 69f6598..b13e13e 100644
--- a/drivers/raw/ifpga/base/meson.build
+++ b/drivers/raw/ifpga/base/meson.build
@@ -25,5 +25,5 @@ sources = [
 
 base_lib = static_library('ifpga_rawdev_base', sources,
 	dependencies: static_rte_eal,
-	c_args: c_args)
+	c_args: cflags)
 base_objs = base_lib.extract_all_objects()
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
index 0ab6fd7..69debff 100644
--- a/drivers/raw/ifpga/meson.build
+++ b/drivers/raw/ifpga/meson.build
@@ -3,18 +3,27 @@
 
 version = 1
 
-subdir('base')
-objs = [base_objs]
-
 dep = dependency('libfdt', required: false)
 if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
 	build = false
 	reason = 'missing dependency, "libfdt"'
 endif
-deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
-	'bus_vdev', 'bus_ifpga', 'net']
-sources = files('ifpga_rawdev.c')
 
-includes += include_directories('base')
+if build
+	subdir('base')
+	objs = [base_objs]
+
+	deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
+		'bus_vdev', 'bus_ifpga', 'net']
+	ext_deps += dep
 
-allow_experimental_apis = true
+	sources = files('ifpga_rawdev.c')
+
+	includes += include_directories('base')
+	includes += include_directories('../../net/ipn3ke')
+
+	allow_experimental_apis = true
+endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 683e3a4..a278552 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -330,7 +330,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 06/19] raw/ifpga/base: align the send buffer for SPI

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 07/19] raw/ifpga/base: add sensor support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 08/19] raw/ifpga/base: introducing sensor APIs

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 09/19] raw/ifpga/base: update SEU register definition

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 10/19] raw/ifpga: add SEU error handler

[Rosen Xu]

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 245 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 1825143..f5e6119 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -27,6 +27,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -605,6 +607,236 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -652,6 +884,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -677,6 +910,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -696,6 +933,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -720,6 +958,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v16 10/19] raw/ifpga: add SEU error handler

[Ferruh Yigit]

On 11/13/2019 7:08 AM, Rosen Xu wrote:
> Add SEU interrupt support for FPGA.
> 
> Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>

<...>

> +static int
> +fme_err_handle_error0(struct opae_manager *mgr)
> +{
> +	struct feature_fme_error0 fme_error0;
> +	u64 val;
> +
> +	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
> +		return -EINVAL;
> +
> +	fme_error0.csr = val;
> +
> +	if (fme_error0.fabric_err)
> +		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
> +	else if (fme_error0.fabfifo_overflow)
> +		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
> +	else if (fme_error0.afu_acc_mode_err)
> +		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
> +	else if (fme_error0.pcie0cdc_parity_err)
> +		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
> +	else if (fme_error0.cvlcdc_parity_err)
> +		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
> +	else if (fme_error0.fpgaseuerr) {
> +		fme_err_read_seu_emr(mgr);
> +		rte_panic("SEU error occurred\n");

Hi Rosen, Andy,

We are not allowed to call 'rte_panic()' from the drivers, can you please remove
all instances?

Re: [dpdk-dev] [PATCH v16 10/19] raw/ifpga: add SEU error handler

[Xu, Rosen]

> -----Original Message-----
> From: Yigit, Ferruh
> Sent: Wednesday, November 13, 2019 22:50
> To: Xu, Rosen <rosen.xu@intel.com>; dev@dpdk.org
> Cc: Zhang, Tianfei <tianfei.zhang@intel.com>; Pei, Andy
> <andy.pei@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>
> Subject: Re: [PATCH v16 10/19] raw/ifpga: add SEU error handler
> 
> On 11/13/2019 7:08 AM, Rosen Xu wrote:
> > Add SEU interrupt support for FPGA.
> >
> > Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
> > Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> 
> <...>
> 
> > +static int
> > +fme_err_handle_error0(struct opae_manager *mgr) {
> > +	struct feature_fme_error0 fme_error0;
> > +	u64 val;
> > +
> > +	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
> > +		return -EINVAL;
> > +
> > +	fme_error0.csr = val;
> > +
> > +	if (fme_error0.fabric_err)
> > +		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
> > +	else if (fme_error0.fabfifo_overflow)
> > +		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow
> error\n");
> > +	else if (fme_error0.afu_acc_mode_err)
> > +		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch
> detected\n");
> > +	else if (fme_error0.pcie0cdc_parity_err)
> > +		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
> > +	else if (fme_error0.cvlcdc_parity_err)
> > +		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
> > +	else if (fme_error0.fpgaseuerr) {
> > +		fme_err_read_seu_emr(mgr);
> > +		rte_panic("SEU error occurred\n");
> 
> Hi Rosen, Andy,
> 
> We are not allowed to call 'rte_panic()' from the drivers, can you please
> remove all instances?

Okay, fixed in v17.

[dpdk-dev] [PATCH v16 11/19] raw/ifpga: add PCIe BDF devices tree scan

[Rosen Xu]

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 551 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 562 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index f5e6119..01ff76a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_malloc.h>
@@ -17,7 +19,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -25,6 +27,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -37,6 +40,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -64,6 +73,494 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(
+		struct ifpga_rawdev *ifpga_rdev, int force_disable);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+		int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
+					sensor->name, value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(
+	struct ifpga_rawdev *ifpga_rdev, int force_disable)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable || force_disable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev,
+					gsd_enable);
+				if (ret == 1 && !gsd_enable) {
+					gsd_enable = 1;
+					i = -1;
+				}
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -372,8 +869,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -842,6 +1340,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -855,7 +1354,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -869,6 +1368,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -987,6 +1494,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1018,13 +1526,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1038,7 +1565,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1065,6 +1593,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 12/19] net/ipn3ke: remove configuration for i40e port bonding

[Rosen Xu]

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/ipn3ke/Makefile                   |   1 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 292 ++++----------------------
 drivers/net/ipn3ke/ipn3ke_flow.c              |   6 +
 drivers/net/ipn3ke/ipn3ke_rawdev_api.h        |  12 ++
 drivers/net/ipn3ke/ipn3ke_representor.c       |  10 +-
 drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map |   6 +
 drivers/raw/ifpga/Makefile                    |   5 +
 drivers/raw/ifpga/ifpga_rawdev.c              |   4 +
 drivers/raw/ifpga/meson.build                 |   5 +-
 9 files changed, 85 insertions(+), 256 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..8c58d71 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,7 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index af87fda..5b5510f 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -35,6 +36,8 @@
 	{ 0, 0 /* sentinel */ },
 };
 
+struct ipn3ke_pub_func ipn3ke_bridge_func;
+
 static int
 ipn3ke_indirect_read(struct ipn3ke_hw *hw, uint32_t *rd_data,
 	uint32_t addr, uint32_t dev_sel, uint32_t eth_group_sel)
@@ -324,7 +327,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -342,6 +346,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -409,9 +419,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -462,7 +469,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -489,7 +500,14 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	if (ipn3ke_bridge_func.get_ifpga_rawdev == NULL)
+		return -ENOMEM;
+	ifpga_dev = ipn3ke_bridge_func.get_ifpga_rawdev(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -501,6 +519,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -508,6 +542,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -553,253 +588,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_flow.c b/drivers/net/ipn3ke/ipn3ke_flow.c
index 9fc3c8b..f857e64 100644
--- a/drivers/net/ipn3ke/ipn3ke_flow.c
+++ b/drivers/net/ipn3ke/ipn3ke_flow.c
@@ -18,6 +18,12 @@
 #include <rte_malloc.h>
 #include <rte_eth_ctrl.h>
 #include <rte_tailq.h>
+#include <rte_rawdev.h>
+#include <rte_rawdev_pmd.h>
+#include <rte_bus_ifpga.h>
+#include <ifpga_common.h>
+#include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
diff --git a/drivers/net/ipn3ke/ipn3ke_rawdev_api.h b/drivers/net/ipn3ke/ipn3ke_rawdev_api.h
index 671fae8..fd2393f 100644
--- a/drivers/net/ipn3ke/ipn3ke_rawdev_api.h
+++ b/drivers/net/ipn3ke/ipn3ke_rawdev_api.h
@@ -59,4 +59,16 @@ struct ifpga_rawdevg_link_info {
 	enum ifpga_rawdev_link_speed link_speed;
 };
 
+struct ipn3ke_pub_func {
+	struct ifpga_rawdev *(*get_ifpga_rawdev)(const struct rte_rawdev *rdv);
+	int (*set_i40e_sw_dev)(uint16_t port_id, struct rte_eth_dev *sw_dev);
+};
+
+/**
+ * @internal
+ * The publid functions of bridge PAC N3000 FPGA and I40e.
+ */
+extern struct ipn3ke_pub_func ipn3ke_bridge_func;
+
+
 #endif /* _IFPGA_RAWDEV_H_ */
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index d37f5e2..8d9ebef 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -2914,12 +2914,18 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	if (representor_param->port_id >= representor_param->hw->port_num)
 		return -ENODEV;
 
+	if (ipn3ke_bridge_func.set_i40e_sw_dev == NULL)
+		return -ENOMEM;
+
 	rpst->ethdev = ethdev;
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		ipn3ke_bridge_func.set_i40e_sw_dev(rpst->i40e_pf_eth_port_id,
+					    rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
diff --git a/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map b/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
index fc8c95e..fbb74ee 100644
--- a/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
+++ b/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
@@ -2,3 +2,9 @@ DPDK_19.05 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	ipn3ke_bridge_func;
+} DPDK_19.05;
diff --git a/drivers/raw/ifpga/Makefile b/drivers/raw/ifpga/Makefile
index 655b292..8fcdb09 100644
--- a/drivers/raw/ifpga/Makefile
+++ b/drivers/raw/ifpga/Makefile
@@ -13,6 +13,7 @@ CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
 CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga_rawdev
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 CFLAGS += -I$(RTE_SDK)/drivers/net/ipn3ke
 LDLIBS += -lrte_eal
 LDLIBS += -lrte_rawdev
@@ -20,6 +21,10 @@ LDLIBS += -lrte_bus_vdev
 LDLIBS += -lrte_kvargs
 LDLIBS += -lrte_bus_pci
 LDLIBS += -lrte_bus_ifpga
+LDLIBS += -lpthread
+LDLIBS += -lfdt
+LDLIBS += -lrte_pmd_i40e
+LDLIBS += -lrte_pmd_ipn3ke
 
 EXPORT_MAP := rte_rawdev_ifpga_version.map
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 01ff76a..b79403e 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -28,6 +28,7 @@
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
 #include <rte_string_fns.h>
+#include <rte_pmd_i40e.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -1368,6 +1369,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	ipn3ke_bridge_func.get_ifpga_rawdev = ifpga_rawdev_get;
+	ipn3ke_bridge_func.set_i40e_sw_dev = rte_pmd_i40e_set_switch_dev;
+
 	dev = ifpga_rawdev_allocate(rawdev);
 	if (dev == NULL) {
 		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
index 69debff..c3459e2 100644
--- a/drivers/raw/ifpga/meson.build
+++ b/drivers/raw/ifpga/meson.build
@@ -16,14 +16,15 @@ if build
 	subdir('base')
 	objs = [base_objs]
 
-	deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
-		'bus_vdev', 'bus_ifpga', 'net']
+	deps += ['ethdev', 'rawdev', 'pci', 'bus_pci', 'kvargs',
+		'bus_vdev', 'bus_ifpga', 'net', 'i40e', 'ipn3ke']
 	ext_deps += dep
 
 	sources = files('ifpga_rawdev.c')
 
 	includes += include_directories('base')
 	includes += include_directories('../../net/ipn3ke')
+	includes += include_directories('../../net/i40e')
 
 	allow_experimental_apis = true
 endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 13/19] raw/ifpga/base: add secure support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 137 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 ++++++++++++-----
 4 files changed, 198 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..748ab56 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,63 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager")) {
+			sysmgr_offset = offset;
+			break;
+		}
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +576,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 14/19] raw/ifpga/base: configure FEC mode

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 15/19] raw/ifpga/base: clean fme errors

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occurred.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 5d6d630..5905eac 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index b79403e..9c9da58 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1175,6 +1175,25 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1184,6 +1203,9 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 16/19] raw/ifpga/base: add new API get board info

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..9f0147d 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disaggregate;
+	u8 lightweight;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..799d67d 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweight = id.lightweiht;
+	fme->board_info.disaggregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweight:%s\n"
+			"disaggregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweight),
+			check_support(fme->board_info.disaggregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 17/19] raw/ifpga: add lightweight fpga image support

[Rosen Xu]

From: Andy Pei <andy.pei@intel.com>

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 9c9da58..5701e2d 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -836,6 +836,8 @@ static int set_surprise_link_check_aer(
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -862,22 +864,40 @@ static int set_surprise_link_check_aer(
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweight) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
-		uuid.b + 8, sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 18/19] raw/ifpga/base: add multiple cards support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, there is one MAX10 chip in each card, and
all of the sensors are connected to MAX10 chip. To support multiple
cards in one server, we introducing a sensor device list under
intel_max10_device instead of a global list. On the other hand, we
using separate intel_max10_device instance for each opae_adatper.

Add mutex lock on do_transaction() function for SPI driver to avoid
race condition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c            |  40 +++++++---
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          |  14 ++--
 drivers/raw/ifpga/base/opae_hw_api.h          |  15 ++--
 drivers/raw/ifpga/base/opae_i2c.c             |  44 ++++++++---
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_intel_max10.c     | 110 ++++++++++++++------------
 drivers/raw/ifpga/base/opae_intel_max10.h     |  19 +++--
 drivers/raw/ifpga/base/opae_spi.c             |   5 --
 drivers/raw/ifpga/base/opae_spi.h             |   3 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  44 +++++++++--
 drivers/raw/ifpga/ifpga_rawdev.c              |  14 ++--
 12 files changed, 205 insertions(+), 109 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 799d67d..c31a94c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -839,8 +839,13 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	struct ifpga_hw *hw;
 	u32 val;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
@@ -854,7 +859,10 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	fme->board_info.seu = id.seu;
 	fme->board_info.ptp = id.ptp;
 
-	dev_info(fme, "found: board: %s type: %s\n",
+	dev_info(fme, "found: PCI dev: %02x:%02x:%x board: %s type: %s\n",
+			hw->pci_data->bus,
+			hw->pci_data->devid,
+			hw->pci_data->function,
 			board_major_to_string(fme->board_info.major),
 			board_type_to_string(fme->board_info.type));
 
@@ -882,11 +890,11 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
-	if (max10_sys_read(MAX10_BUILD_VER, &val))
+	if (max10_sys_read(fme->max10_dev, MAX10_BUILD_VER, &val))
 		return -EINVAL;
 	fme->board_info.max10_version = val & 0xffffff;
 
-	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+	if (max10_sys_read(fme->max10_dev, NIOS2_FW_VERSION, &val))
 		return -EINVAL;
 	fme->board_info.nios_fw_version = val & 0xffffff;
 
@@ -897,12 +905,12 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	return 0;
 }
 
-static int spi_self_checking(void)
+static int spi_self_checking(struct intel_max10_device *dev)
 {
 	u32 val;
 	int ret;
 
-	ret = max10_sys_read(MAX10_TEST_REG, &val);
+	ret = max10_sys_read(dev, MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
@@ -937,10 +945,11 @@ static int fme_spi_init(struct ifpga_feature *feature)
 		goto spi_fail;
 	}
 
+
 	fme->max10_dev = max10;
 
 	/* SPI self test */
-	if (spi_self_checking()) {
+	if (spi_self_checking(max10)) {
 		ret = -EIO;
 		goto max10_fail;
 	}
@@ -1041,8 +1050,18 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 	struct ifpga_fme_hw *fme = (struct ifpga_fme_hw *)feature->parent;
 	struct altera_spi_device *spi_master;
 	struct intel_max10_device *max10;
+	struct ifpga_hw *hw;
+	struct opae_manager *mgr;
 	int ret = 0;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
+	mgr = hw->adapter->mgr;
+	if (!mgr)
+		return -ENODEV;
+
 	dev_info(fme, "FME SPI Master (NIOS) Init.\n");
 	dev_debug(fme, "FME SPI base addr %p.\n",
 			feature->addr);
@@ -1080,12 +1099,15 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 		goto release_dev;
 	}
 
+	max10->bus = hw->pci_data->bus;
+
 	fme_get_board_interface(fme);
 
 	fme->max10_dev = max10;
+	mgr->sensor_list = &max10->opae_sensor_list;
 
 	/* SPI self test */
-	if (spi_self_checking())
+	if (spi_self_checking(max10))
 		goto spi_fail;
 
 	return ret;
@@ -1344,7 +1366,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(dev, PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1372,7 +1394,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(sensor->value_reg, value)) {
+	if (max10_sys_read(dev, sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_debug.c b/drivers/raw/ifpga/base/opae_debug.c
index 88f2d5c..dad3ea3 100644
--- a/drivers/raw/ifpga/base/opae_debug.c
+++ b/drivers/raw/ifpga/base/opae_debug.c
@@ -59,6 +59,9 @@ static void opae_adapter_data_dump(void *data)
 		opae_log("OPAE Adapter Type = PCI\n");
 		opae_log("PCI Device ID: 0x%04x\n", d_pci->device_id);
 		opae_log("PCI Vendor ID: 0x%04x\n", d_pci->vendor_id);
+		opae_log("PCI bus: 0x%04x\n", d_pci->bus);
+		opae_log("PCI devid: 0x%04x\n", d_pci->devid);
+		opae_log("PCI function: 0x%04x\n", d_pci->function);
 
 		for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 			r = &d_pci->region[i];
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 1ccc967..c969dfe 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -583,11 +583,12 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_id(unsigned int id)
+opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (sensor->id == id)
 			return sensor;
 
@@ -601,11 +602,12 @@ struct opae_sensor_info *
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_name(const char *name)
+opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (!strcmp(sensor->name, name))
 			return sensor;
 
@@ -630,7 +632,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_name(name);
+	sensor = opae_mgr_get_sensor_by_name(mgr, name);
 	if (!sensor)
 		return -ENODEV;
 
@@ -658,7 +660,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_id(id);
+	sensor = opae_mgr_get_sensor_by_id(mgr, id);
 	if (!sensor)
 		return -ENODEV;
 
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index b78fbd5..cdf369f 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -40,6 +40,7 @@ struct opae_manager {
 	struct opae_adapter *adapter;
 	struct opae_manager_ops *ops;
 	struct opae_manager_networking_ops *network_ops;
+	struct opae_sensor_list *sensor_list;
 	void *data;
 };
 
@@ -75,9 +76,8 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
-extern struct opae_sensor_list opae_sensor_list;
-#define opae_mgr_for_each_sensor(sensor) \
-	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+#define opae_mgr_for_each_sensor(mgr, sensor) \
+	TAILQ_FOREACH(sensor, mgr->sensor_list, node)
 
 /* OPAE Manager APIs */
 struct opae_manager *
@@ -88,8 +88,10 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
-struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
-struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id);
 int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
 		const char *name, unsigned int *value);
 int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
@@ -241,6 +243,9 @@ struct opae_adapter_data_pci {
 	enum opae_adapter_type type;
 	u16 device_id;
 	u16 vendor_id;
+	u16 bus; /*Device bus for PCI */
+	u16 devid; /* Device ID */
+	u16 function; /* Device function */
 	struct opae_reg_region region[PCI_MAX_RESOURCE];
 	int vfio_dev_fd;  /* VFIO device file descriptor */
 };
diff --git a/drivers/raw/ifpga/base/opae_i2c.c b/drivers/raw/ifpga/base/opae_i2c.c
index f8bc247..846d751 100644
--- a/drivers/raw/ifpga/base/opae_i2c.c
+++ b/drivers/raw/ifpga/base/opae_i2c.c
@@ -28,6 +28,9 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 {
 	u8 msgbuf[2];
 	int i = 0;
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
 
 	if (flags & I2C_FLAG_ADDR16)
 		msgbuf[i++] = offset >> 8;
@@ -49,10 +52,16 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		},
 	};
 
-	if (!dev->xfer)
-		return -ENODEV;
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	ret = i2c_transfer(dev, msg, 2);
 
-	return i2c_transfer(dev, msg, 2);
+exit:
+	pthread_mutex_unlock(&dev->lock);
+	return ret;
 }
 
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
@@ -63,12 +72,18 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	int ret;
 	int i = 0;
 
-	if (!dev->xfer)
-		return -ENODEV;
+	pthread_mutex_lock(&dev->lock);
+
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
 
 	buf = opae_malloc(I2C_MAX_OFFSET_LEN + len);
-	if (!buf)
-		return -ENOMEM;
+	if (!buf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
 
 	msg.addr = slave_addr;
 	msg.flags = 0;
@@ -84,6 +99,8 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	ret = i2c_transfer(dev, &msg, 1);
 
 	opae_free(buf);
+exit:
+	pthread_mutex_unlock(&dev->lock);
 	return ret;
 }
 
@@ -477,14 +494,19 @@ struct altera_i2c_dev *altera_i2c_probe(void *base)
 	dev->i2c_clk = dev->i2c_param.ref_clk * 1000000;
 	dev->xfer = altera_i2c_xfer;
 
+	if (pthread_mutex_init(&dev->lock, NULL))
+		return NULL;
+
 	altera_i2c_hardware_init(dev);
 
 	return dev;
 }
 
-int altera_i2c_remove(struct altera_i2c_dev *dev)
+void altera_i2c_remove(struct altera_i2c_dev *dev)
 {
-	altera_i2c_disable(dev);
-
-	return 0;
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
+		altera_i2c_disable(dev);
+		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/base/opae_i2c.h b/drivers/raw/ifpga/base/opae_i2c.h
index 8890c8f..266e127 100644
--- a/drivers/raw/ifpga/base/opae_i2c.h
+++ b/drivers/raw/ifpga/base/opae_i2c.h
@@ -93,6 +93,7 @@ struct altera_i2c_dev {
 	u32 isr_mask;
 	u8 *buf;
 	int (*xfer)(struct altera_i2c_dev *dev, struct i2c_msg *msg, int num);
+	pthread_mutex_t lock;
 };
 
 /**
@@ -114,7 +115,7 @@ enum i2c_msg_flags {
 };
 
 struct altera_i2c_dev *altera_i2c_probe(void *base);
-int altera_i2c_remove(struct altera_i2c_dev *dev);
+void altera_i2c_remove(struct altera_i2c_dev *dev);
 int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		u32 offset, u8 *buf, u32 count);
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 748ab56..8e23ca1 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -5,45 +5,50 @@
 #include "opae_intel_max10.h"
 #include <libfdt.h>
 
-static struct intel_max10_device *g_max10;
-
-struct opae_sensor_list opae_sensor_list =
-	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
-
-int max10_reg_read(unsigned int reg, unsigned int *val)
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_read(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x\n", __func__, dev->bus, reg);
+
+	return spi_transaction_read(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)val);
 }
 
-int max10_reg_write(unsigned int reg, unsigned int val)
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val)
 {
 	unsigned int tmp = val;
 
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_write(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x, val:0x%x\n", __func__,
+			dev->bus, reg, val);
+
+	return spi_transaction_write(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)&tmp);
 }
 
-int max10_sys_read(unsigned int offset, unsigned int *val)
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_read(g_max10->base + offset, val);
+
+	return max10_reg_read(dev, dev->base + offset, val);
 }
 
-int max10_sys_write(unsigned int offset, unsigned int val)
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_write(g_max10->base + offset, val);
+	return max10_reg_write(dev, dev->base + offset, val);
 }
 
 static struct max10_compatible_id max10_id_table[] = {
@@ -86,8 +91,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
-static int altera_nor_flash_read(u32 offset,
-		void *buffer, u32 len)
+static int altera_nor_flash_read(struct intel_max10_device *dev,
+	u32 offset, void *buffer, u32 len)
 {
 	int word_len;
 	int i;
@@ -95,13 +100,13 @@ static int altera_nor_flash_read(u32 offset,
 	unsigned int value;
 	int ret;
 
-	if (!buffer || len <= 0)
+	if (!dev || !buffer || len <= 0)
 		return -ENODEV;
 
 	word_len = len/4;
 
 	for (i = 0; i < word_len; i++) {
-		ret = max10_reg_read(offset + i*4,
+		ret = max10_reg_read(dev, offset + i*4,
 				&value);
 		if (ret)
 			return -EBUSY;
@@ -112,12 +117,12 @@ static int altera_nor_flash_read(u32 offset,
 	return 0;
 }
 
-static int enable_nor_flash(bool on)
+static int enable_nor_flash(struct intel_max10_device *dev, bool on)
 {
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_sys_read(RSU_REG, &val);
+	ret = max10_sys_read(dev, RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -128,7 +133,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_sys_write(RSU_REG, val);
+	return max10_sys_write(dev, RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -140,7 +145,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_sys_read(DT_AVAIL_REG, &val);
+	ret = max10_sys_read(max10, DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -151,19 +156,19 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
+	ret = max10_sys_read(max10, DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
 	}
 
-	ret = enable_nor_flash(true);
+	ret = enable_nor_flash(max10, true);
 	if (ret) {
 		dev_err(max10 "fail to enable flash\n");
 		return ret;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	ret = altera_nor_flash_read(max10, dt_addr, &hdr, sizeof(hdr));
 	if (ret) {
 		dev_err(max10 "read fdt header fail\n");
 		goto done;
@@ -188,7 +193,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		goto done;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	ret = altera_nor_flash_read(max10, dt_addr, fdt_root, dt_size);
 	if (ret) {
 		dev_err(max10 "cannot read device table\n");
 		goto done;
@@ -207,7 +212,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	max10->fdt_root = fdt_root;
 
 done:
-	ret = enable_nor_flash(false);
+	ret = enable_nor_flash(max10, false);
 
 	if (ret && fdt_root)
 		opae_free(fdt_root);
@@ -298,12 +303,12 @@ static int fdt_get_named_reg(const void *fdt, int node, const char *name,
 	return fdt_get_reg(fdt, node, idx, start, size);
 }
 
-static void max10_sensor_uinit(void)
+static void max10_sensor_uinit(struct intel_max10_device *dev)
 {
 	struct opae_sensor_info *info;
 
-	TAILQ_FOREACH(info, &opae_sensor_list, node) {
-		TAILQ_REMOVE(&opae_sensor_list, info, node);
+	TAILQ_FOREACH(info, &dev->opae_sensor_list, node) {
+		TAILQ_REMOVE(&dev->opae_sensor_list, info, node);
 		opae_free(info);
 	}
 }
@@ -313,8 +318,8 @@ static bool sensor_reg_valid(struct sensor_reg *reg)
 	return !!reg->size;
 }
 
-static int max10_add_sensor(struct raw_sensor_info *info,
-		struct opae_sensor_info *sensor)
+static int max10_add_sensor(struct intel_max10_device *dev,
+	struct raw_sensor_info *info, struct opae_sensor_info *sensor)
 {
 	int i;
 	int ret = 0;
@@ -332,12 +337,15 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_sys_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(dev, info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
-		if (val == 0xdeadbeef)
+		if (val == 0xdeadbeef) {
+			dev_debug(dev, "%s: sensor:%s invalid 0x%x at:%d\n",
+				__func__, sensor->name, val, i);
 			continue;
+		}
 
 		val *= info->multiplier;
 
@@ -458,7 +466,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
 		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
 
-		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+		dev_debug(dev, "found sensor from DTB: %s: %s: %u: %u\n",
 				raw->name, raw->type,
 				raw->id, raw->multiplier);
 
@@ -473,15 +481,16 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 			goto free_sensor;
 		}
 
-		if (max10_add_sensor(raw, sensor)) {
+		if (max10_add_sensor(dev, raw, sensor)) {
 			ret = -EINVAL;
 			opae_free(sensor);
 			goto free_sensor;
 		}
 
-		if (sensor->flags & OPAE_SENSOR_VALID)
-			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
-		else
+		if (sensor->flags & OPAE_SENSOR_VALID) {
+			TAILQ_INSERT_TAIL(&dev->opae_sensor_list, sensor, node);
+			dev_info(dev, "found valid sensor: %s\n", sensor->name);
+		} else
 			opae_free(sensor);
 
 		opae_free(raw);
@@ -492,7 +501,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 free_sensor:
 	if (raw)
 		opae_free(raw);
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 	return ret;
 }
 
@@ -500,7 +509,7 @@ static int check_max10_version(struct intel_max10_device *dev)
 {
 	unsigned int v;
 
-	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+	if (!max10_reg_read(dev, MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
 				&v)) {
 		if (v != 0xffffffff) {
 			dev_info(dev, "secure MAX10 detected\n");
@@ -565,6 +574,8 @@ struct intel_max10_device *
 	if (!dev)
 		return NULL;
 
+	TAILQ_INIT(&dev->opae_sensor_list);
+
 	dev->spi_master = spi;
 
 	dev->spi_tran_dev = spi_transaction_init(spi, chipselect);
@@ -573,9 +584,6 @@ struct intel_max10_device *
 		goto free_dev;
 	}
 
-	/* set the max10 device firstly */
-	g_max10 = dev;
-
 	/* check the max10 version */
 	ret = check_max10_version(dev);
 	if (ret) {
@@ -601,7 +609,7 @@ struct intel_max10_device *
 	}
 
 	/* read FPGA loading information */
-	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
+	ret = max10_sys_read(dev, FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
 		goto release_max10_hw;
@@ -611,14 +619,13 @@ struct intel_max10_device *
 	return dev;
 
 release_max10_hw:
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return NULL;
@@ -629,7 +636,7 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
@@ -637,7 +644,6 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index e632941..123cdc4 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -26,6 +26,9 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SECURE		BIT(6)
 #define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
@@ -33,6 +36,8 @@ struct intel_max10_device {
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
 	unsigned int base; /* max10 base address */
+	u16 bus;
+	struct opae_sensor_list opae_sensor_list;
 };
 
 /* retimer speed */
@@ -136,17 +141,19 @@ struct opae_retimer_status {
 
 #define DFT_MAX_SIZE		0x7e0000
 
-int max10_reg_read(unsigned int reg, unsigned int *val);
-int max10_reg_write(unsigned int reg, unsigned int val);
-int max10_sys_read(unsigned int offset, unsigned int *val);
-int max10_sys_write(unsigned int offset, unsigned int val);
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val);
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val);
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val);
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
-/** List of opae sensors */
-TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
 
 #define SENSOR_REG_VALUE 0x0
 #define SENSOR_REG_HIGH_WARN 0x1
diff --git a/drivers/raw/ifpga/base/opae_spi.c b/drivers/raw/ifpga/base/opae_spi.c
index cc52782..bfdc83e 100644
--- a/drivers/raw/ifpga/base/opae_spi.c
+++ b/drivers/raw/ifpga/base/opae_spi.c
@@ -181,7 +181,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 	u32 rxd;
 	unsigned int tx_data;
 	u32 status;
-	int retry = 0;
 	int ret;
 
 	while (count < dev->len) {
@@ -194,10 +193,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 				return -EIO;
 			if (status & ALTERA_SPI_STATUS_RRDY_MSK)
 				break;
-			if (retry++ > SPI_MAX_RETRY) {
-				dev_err(dev, "%s, read timeout\n", __func__);
-				return -EBUSY;
-			}
 		}
 
 		ret = spi_reg_read(dev, ALTERA_SPI_RXDATA, &rxd);
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index 6355deb..d20a4c3 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -38,7 +38,7 @@
 #define SPI_WRITE 0x20
 #define WRITE_DATA_MASK GENMASK_ULL(31, 0)
 
-#define SPI_MAX_RETRY 100000
+#define SPI_MAX_RETRY 1000000
 
 #define TYPE_SPI 0
 #define TYPE_NIOS_SPI 1
@@ -102,6 +102,7 @@ struct spi_transaction_dev {
 	struct altera_spi_device *dev;
 	int chipselect;
 	struct spi_tran_buffer *buffer;
+	pthread_mutex_t lock;
 };
 
 struct spi_tran_header {
diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 06ca625..013efee 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -154,6 +154,8 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	unsigned int rx_len = 0;
 	int retry = 0;
 	int spi_flags;
+	unsigned long timeout = msecs_to_timer_cycles(1000);
+	unsigned long ticks;
 	unsigned int resp_max_len = 2 * resp_len;
 
 	print_buffer("before bytes:", send_data, send_len);
@@ -207,8 +209,11 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	if (ret != SPI_FOUND_EOP) {
 		tx_buffer = NULL;
 		tx_len = 0;
-		if (retry++ > 10) {
-			dev_err(NULL, "cannot found valid data from SPI\n");
+		ticks = rte_get_timer_cycles();
+		if (time_after(ticks, timeout) &&
+				retry++ > SPI_MAX_RETRY) {
+			dev_err(NULL, "Have retry %d, found invalid packet data\n",
+				retry);
 			return -EBUSY;
 		}
 
@@ -427,23 +432,36 @@ static int do_transaction(struct spi_transaction_dev *dev, unsigned int addr,
 int spi_transaction_read(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_READ : SPI_TRAN_NON_SEQ_READ);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 int spi_transaction_write(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_WRITE : SPI_TRAN_NON_SEQ_WRITE);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 		int chipselect)
 {
 	struct spi_transaction_dev *spi_tran_dev;
+	int ret;
 
 	spi_tran_dev = opae_malloc(sizeof(struct spi_transaction_dev));
 	if (!spi_tran_dev)
@@ -453,18 +471,28 @@ struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 	spi_tran_dev->chipselect = chipselect;
 
 	spi_tran_dev->buffer = opae_malloc(sizeof(struct spi_tran_buffer));
-	if (!spi_tran_dev->buffer) {
-		opae_free(spi_tran_dev);
-		return NULL;
+	if (!spi_tran_dev->buffer)
+		goto err;
+
+	ret = pthread_mutex_init(&spi_tran_dev->lock, NULL);
+	if (ret) {
+		dev_err(spi_tran_dev, "fail to init mutex lock\n");
+		goto err;
 	}
 
 	return spi_tran_dev;
+
+err:
+	opae_free(spi_tran_dev);
+	return NULL;
 }
 
 void spi_transaction_remove(struct spi_transaction_dev *dev)
 {
 	if (dev && dev->buffer)
 		opae_free(dev->buffer);
-	if (dev)
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
 		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 5701e2d..88d632d 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -361,7 +361,7 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 	if (!mgr)
 		return -ENODEV;
 
-	opae_mgr_for_each_sensor(sensor) {
+	opae_mgr_for_each_sensor(mgr, sensor) {
 		if (!(sensor->flags & OPAE_SENSOR_VALID))
 			goto fail;
 
@@ -370,8 +370,8 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 			goto fail;
 
 		if (value == 0xdeadbeef) {
-			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
-					sensor->name, value);
+			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value %x\n",
+					raw_dev->dev_id, sensor->name, value);
 			continue;
 		}
 
@@ -394,8 +394,9 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 		/* monitor 12V AUX sensor */
 		if (!strcmp(sensor->name, "12V AUX Voltage")) {
 			if (value < AUX_VOLTAGE_WARN) {
-				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
-						sensor->name, value);
+				IFPGA_RAWDEV_PMD_INFO(
+					"%s reach theshold %d mV\n",
+					sensor->name, value);
 				*gsd_start = true;
 				break;
 			}
@@ -1437,6 +1438,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->bus = pci_dev->addr.bus;
+	data->devid = pci_dev->addr.devid;
+	data->function = pci_dev->addr.function;
 	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
-- 
1.8.3.1

[dpdk-dev] [PATCH v16 19/19] raw/ifpga: introducing new irq API

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing new register and unregister API for ifpga interrupt.
1. register FME and AFU interrupt
   ifpga_register_msix_irq()

2. unregister FME and AFU interrupt
   ifpga_unregister_msix_irq()

On PAC N3000 card, there is one PCIe MSIX interrupt for FME managerment,
like the error report, thermal management, we use this interrupt in
ifpga_rawdev device driver. on the other hand, there are about 4 PCIe
MSIX interrupts are reserved for AFU which end-user can use those
interrupts in their AFU logic design. End-user can use those APIs to
register interrupt handler in their AFU drivers.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 105 +++++++++++++++++++++++++++------------
 drivers/raw/ifpga/ifpga_rawdev.h |  14 ++++++
 2 files changed, 88 insertions(+), 31 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 88d632d..c930710 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -79,6 +79,10 @@
 static int ifpga_monitor_start;
 static pthread_t ifpga_monitor_start_thread;
 
+#define IFPGA_MAX_IRQ 12
+/* 0 for FME interrupt, others are reserved for AFU irq */
+static struct rte_intr_handle ifpga_irq_handle[IFPGA_MAX_IRQ];
+
 static struct ifpga_rawdev *
 ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
 static int set_surprise_link_check_aer(
@@ -1332,53 +1336,90 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 	fme_err_handle_catfatal_error(mgr);
 }
 
-static struct rte_intr_handle fme_intr_handle;
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg)
+{
+	struct rte_intr_handle intr_handle;
+
+	if (type == IFPGA_FME_IRQ)
+		intr_handle = ifpga_irq_handle[0];
+	else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+	rte_intr_efd_disable(&intr_handle);
+
+	return rte_intr_callback_unregister(&intr_handle,
+			handler, arg);
+}
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg)
 {
 	int ret;
-	struct fpga_fme_err_irq_set err_irq_set;
+	struct rte_intr_handle intr_handle;
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_accelerator *acc;
 
-	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+	adapter = ifpga_rawdev_get_priv(dev);
+	if (!adapter)
+		return -ENODEV;
 
-	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
-	if (ret)
-		return -EINVAL;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
 
-	fme_intr_handle.fd = fme_intr_handle.efds[0];
+	if (type == IFPGA_FME_IRQ) {
+		intr_handle = ifpga_irq_handle[0];
+		count = 1;
+	} else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
-			fme_intr_handle.vfio_dev_fd,
-			fme_intr_handle.efds[0], fme_intr_handle.fd);
+	intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
 
-	err_irq_set.evtfd = fme_intr_handle.efds[0];
-	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	ret = rte_intr_efd_enable(&intr_handle, count);
 	if (ret)
-		return -EINVAL;
+		return -ENODEV;
+
+	intr_handle.fd = intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("register %s irq, vfio_fd=%d, fd=%d\n",
+			name, intr_handle.vfio_dev_fd,
+			intr_handle.fd);
+
+	if (type == IFPGA_FME_IRQ) {
+		struct fpga_fme_err_irq_set err_irq_set;
+		err_irq_set.evtfd = intr_handle.efds[0];
+
+		ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+		if (ret)
+			return -EINVAL;
+	} else if (type == IFPGA_AFU_IRQ) {
+		acc = opae_adapter_get_acc(adapter, port_id);
+		if (!acc)
+			return -EINVAL;
 
-	/* register FME interrupt using DPDK API */
-	ret = rte_intr_callback_register(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
+		ret = opae_acc_set_irq(acc, vec_start, count, intr_handle.efds);
+		if (ret)
+			return -EINVAL;
+	}
+
+	/* register interrupt handler using DPDK API */
+	ret = rte_intr_callback_register(&intr_handle,
+			handler, (void *)arg);
 	if (ret)
 		return -EINVAL;
 
-	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+	IFPGA_RAWDEV_PMD_INFO("success register %s interrupt\n", name);
 
 	return 0;
 }
 
 static int
-ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
-{
-	rte_intr_efd_disable(&fme_intr_handle);
-
-	return rte_intr_callback_unregister(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
-}
-
-static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -1467,7 +1508,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
-	ret = ifpga_register_fme_interrupt(mgr);
+	ret = ifpga_register_msix_irq(rawdev, 0, IFPGA_FME_IRQ, 0, 0,
+			fme_interrupt_handler, "fme_irq", mgr);
 	if (ret)
 		goto free_adapter_data;
 
@@ -1519,7 +1561,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	if (!mgr)
 		return -ENODEV;
 
-	if (ifpga_unregister_fme_interrupt(mgr))
+	if (ifpga_unregister_msix_irq(IFPGA_FME_IRQ, 0,
+				fme_interrupt_handler, mgr))
 		return -EINVAL;
 
 	opae_adapter_data_free(adapter->data);
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index bd42083..7754beb 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -62,4 +62,18 @@ struct ifpga_rawdev {
 struct ifpga_rawdev *
 ifpga_rawdev_get(const struct rte_rawdev *rawdev);
 
+enum ifpga_irq_type {
+	IFPGA_FME_IRQ = 0,
+	IFPGA_AFU_IRQ = 1,
+};
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg);
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Rosen Xu]

This patch set adds PCIe AER disable and FPGA interrupt support for
ipn3ke. It also provides a small rework for port bonding between FPGA
line side port and I40e PF port.

What is the PCI Express AER(Advanced Error Reporting)?
Advanced Error Reporting capability is implemented with a PCI Express
advanced error reporting extended capability structure providing more
robust error reporting. It's also one of PCI Express error reporting
paradigms. AER is supported by most of PCIe devices.

In PAC N3000 card, some uncertainty errors will cause FPGA reload,
such as temperature is higher than threshold. From Software point of
view, FPGA reload means FPGA unplug and plug. For avoiding system
crash we need to clear AER register before these errors occur.

Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga
rawdev driver, we implement a FME interrupt function to notify errors
reported by FME. Besides this, OPAE share code also provide a common
AFU interrupt API for users to register their own interrupt functions.

v17 updates:
=========
- remove rte_panic.

v16 updates:
=========
- rebase to DPDK 19.11 RC-2.

v15 updates:
=========
- fix share library build issues.

v14 updates:
=========
- fix coding style and remove unnecessary comments.

v13 updates:
=========
- fix meson build issue.

v12 updates:
=========
- fix meson build issue.
 
v11 updates:
=========
- move symbol ifpga_rawdev_ge to the EXPERIMENTAL section of 
  drivers/raw/ifpga/rte_rawdev_ifpga_version.map

v10 updates:
=========
- introducing new irq API
- fix meson build issue

v9 updates:
=========
- Add mutex lock on do_transaction() function for SPI driver to avoid race
  condition.

v8 updates:
=========
- add multiple cards support.

v7 updates:
==========
- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
- function rte_pmd_i40e_set_switch_dev works as an external API,
  use port_id but not rte_eth_dev as parameter.
- add doxygen header here for the new API.
- update the rte_pmd_i40e_version.map.
- fix coding style issue.
- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
- for functions with a lot of similarity, extract out common function to reduce
  duplication.

v6 updates:
=========
- correct author information.
- correct typo in commit message and remove Gerrit Change-Id's before
  submitting upstream

v5 updates:
==========
- add lightweight fpga image support. in lightweight fpga image mode,
  ipn3ke representor will not be probed.

v4 updates:
===========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support

Andy Pei (2):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga: add lightweight fpga image support

Rosen Xu (3):
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei zhang (14):
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga/base: add secure support
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors
  raw/ifpga/base: add new API get board info
  raw/ifpga/base: add multiple cards support
  raw/ifpga: introducing new irq API

 config/common_base                            |   4 +-
 config/common_linux                           |   6 +
 drivers/meson.build                           |   6 +-
 drivers/net/i40e/base/i40e_type.h             |   3 +
 drivers/net/i40e/i40e_ethdev.c                |   6 +
 drivers/net/i40e/rte_pmd_i40e.c               |  21 +
 drivers/net/i40e/rte_pmd_i40e.h               |  18 +
 drivers/net/i40e/rte_pmd_i40e_version.map     |   8 +-
 drivers/net/ipn3ke/Makefile                   |   1 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 292 ++-------
 drivers/net/ipn3ke/ipn3ke_flow.c              |   6 +
 drivers/net/ipn3ke/ipn3ke_rawdev_api.h        |  12 +
 drivers/net/ipn3ke/ipn3ke_representor.c       |  10 +-
 drivers/net/ipn3ke/meson.build                |  24 +-
 drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map |   6 +
 drivers/raw/ifpga/Makefile                    |   5 +
 drivers/raw/ifpga/base/ifpga_api.c            |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h        |  75 ++-
 drivers/raw/ifpga/base/ifpga_feature_dev.c    |  59 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c            | 166 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c      |  74 ++-
 drivers/raw/ifpga/base/ifpga_hw.h             |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c           |  18 +
 drivers/raw/ifpga/base/ifpga_port_error.c     |  19 +
 drivers/raw/ifpga/base/meson.build            |   2 +-
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          | 137 ++++
 drivers/raw/ifpga/base/opae_hw_api.h          |  26 +
 drivers/raw/ifpga/base/opae_i2c.c             |  44 +-
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c     | 599 ++++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h     | 157 ++++-
 drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
 drivers/raw/ifpga/base/opae_spi.c             |   5 -
 drivers/raw/ifpga/base/opae_spi.h             |  26 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  84 ++-
 drivers/raw/ifpga/ifpga_rawdev.c              | 903 +++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h              |  30 +
 drivers/raw/ifpga/meson.build                 |  26 +-
 mk/rte.app.mk                                 |   2 +-
 42 files changed, 2480 insertions(+), 441 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v17 01/19] net/i40e: i40e support ipn3ke FPGA port bonding

[Rosen Xu]

From: Andy Pei <andy.pei@intel.com>

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h         |  3 +++
 drivers/net/i40e/i40e_ethdev.c            |  6 ++++++
 drivers/net/i40e/rte_pmd_i40e.c           | 21 +++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h           | 18 ++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e_version.map |  8 +++++++-
 5 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 9a81f99..1afddeb 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1393,6 +1393,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2906,6 +2909,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index 4c3c708..fdcb1a4 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -3207,3 +3207,24 @@ int rte_pmd_i40e_flow_add_del_packet_template(
 	I40E_WRITE_FLUSH(hw);
 	return 0;
 }
+
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev)
+{
+	struct rte_eth_dev *i40e_dev;
+	struct i40e_hw *hw;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+	i40e_dev = &rte_eth_devices[port_id];
+	if (!is_i40e_supported(i40e_dev))
+		return -ENOTSUP;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+	if (!hw)
+		return -1;
+
+	hw->switch_dev = switch_dev;
+
+	return 0;
+}
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..915cdf0 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+/**
+ * For ipn3ke, i40e works with FPGA.
+ * In this situation, i40e get link status from fpga,
+ * fpga works as switch_dev for i40e.
+ * This function set switch_dev for i40e.
+ *
+ * @param port_id
+ *    port_id of i40e device to be set switch device.
+ * @param switch_dev
+ *    target switch device from which i40e device to get link status from.
+ * @return
+ *   - (less than 0) if failed.
+ *   - (0) if success.
+ */
+__rte_experimental
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
index cccd576..ca5bcee 100644
--- a/drivers/net/i40e/rte_pmd_i40e_version.map
+++ b/drivers/net/i40e/rte_pmd_i40e_version.map
@@ -64,4 +64,10 @@ DPDK_18.02 {
 
 	rte_pmd_i40e_inset_get;
 	rte_pmd_i40e_inset_set;
-} DPDK_17.11;
\ No newline at end of file
+} DPDK_17.11;
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_i40e_set_switch_dev;
+};
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 02/19] raw/ifpga/base: add irq support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME global error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

To build this feature, CONFIG_RTE_EAL_VFIO should be enabled.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                         |  2 +-
 config/common_linux                        |  6 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 59 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 19 ++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 18 +++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 19 ++++++++++
 6 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/config/common_base b/config/common_base
index 9142778..36ab13d 100644
--- a/config/common_base
+++ b/config/common_base
@@ -788,7 +788,7 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
 #
 # Compile PMD for Intel FPGA raw device
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
 
 #
 # Compile PMD for Intel IOAT raw device
diff --git a/config/common_linux b/config/common_linux
index 96e2e1f..a78b8c6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -68,3 +68,9 @@ CONFIG_RTE_LIBRTE_HINIC_PMD=y
 # Hisilicon HNS3 PMD driver
 #
 CONFIG_RTE_LIBRTE_HNS3_PMD=y
+
+#
+# Compile PMD for Intel FPGA raw device
+# To compile, CONFIG_RTE_EAL_VFIO should be enabled.
+#
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..0f852a7 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,61 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+#define FPGA_MAX_MSIX_VEC_COUNT	128
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MAX_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	/* 0 < irq_set->count < FPGA_MAX_MSIX_VEC_COUNT */
+	irq_set->count = count ?
+		(count > FPGA_MAX_MSIX_VEC_COUNT ?
+		 FPGA_MAX_MSIX_VEC_COUNT : count) : 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..2978c79 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,28 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&fme->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..c0aaf01 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,27 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..189f762 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,28 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 03/19] raw/ifpga/base: clear pending bit

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 2978c79..be041ec 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 04/19] raw/ifpga/base: add SEU error support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 31 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index be041ec..5d6d630 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,33 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr(struct ifpga_fme_hw *fme,
+		u64 *val, bool high)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+				FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	if (high)
+		*val = readq(&fme_err->seu_emr_h);
+	else
+		*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +297,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr(fme, &prop->data, 0);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr(fme, &prop->data, 1);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 05/19] raw/ifpga/base: add device tree support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

To eliminate build error, we currently do not compile raw/ifpga
and net/ipn3ke. User should install libfdt and libfdt-devel first,
modify config/common_linux, CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
to CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y, modify config/common_base,
CONFIG_RTE_LIBRTE_IPN3KE_PMD=n to CONFIG_RTE_LIBRTE_IPN3KE_PMD=y.
Then this function can work.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 config/common_base                        |   2 +-
 config/common_linux                       |   2 +-
 drivers/meson.build                       |   6 +-
 drivers/net/ipn3ke/meson.build            |  24 +++-
 drivers/raw/ifpga/base/meson.build        |   2 +-
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 drivers/raw/ifpga/meson.build             |  25 ++--
 mk/rte.app.mk                             |   2 +-
 9 files changed, 235 insertions(+), 21 deletions(-)

diff --git a/config/common_base b/config/common_base
index 36ab13d..7c015f9 100644
--- a/config/common_base
+++ b/config/common_base
@@ -347,7 +347,7 @@ CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
 #
 # Compile burst-oriented IPN3KE PMD driver
 #
-CONFIG_RTE_LIBRTE_IPN3KE_PMD=y
+CONFIG_RTE_LIBRTE_IPN3KE_PMD=n
 
 #
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
diff --git a/config/common_linux b/config/common_linux
index a78b8c6..c5cf3d6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -73,4 +73,4 @@ CONFIG_RTE_LIBRTE_HNS3_PMD=y
 # Compile PMD for Intel FPGA raw device
 # To compile, CONFIG_RTE_EAL_VFIO should be enabled.
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
diff --git a/drivers/meson.build b/drivers/meson.build
index 823e3ef..b7fdfb7 100644
--- a/drivers/meson.build
+++ b/drivers/meson.build
@@ -9,12 +9,12 @@ endif
 dpdk_driver_classes = ['common',
 	       'bus',
 	       'mempool', # depends on common and bus.
-	       'net',     # depends on common, bus and mempool.
+	       'raw',     # depends on common and bus.
+	       'net',     # depends on common, bus, mempool and raw.
 	       'crypto',  # depends on common, bus and mempool (net in future).
 	       'compress', # depends on common, bus, mempool.
 	       'event',   # depends on common, bus, mempool and net.
-	       'baseband', # depends on common and bus.
-	       'raw']     # depends on common, bus, mempool, net and event.
+	       'baseband'] # depends on common and bus.
 
 disabled_drivers = get_option('disable_drivers').split(',')
 
diff --git a/drivers/net/ipn3ke/meson.build b/drivers/net/ipn3ke/meson.build
index 74b4d7c..e3c8a67 100644
--- a/drivers/net/ipn3ke/meson.build
+++ b/drivers/net/ipn3ke/meson.build
@@ -8,10 +8,22 @@
 #  rte_eth_dev_destroy()
 #  rte_eth_switch_domain_free()
 #
-allow_experimental_apis = true
 
-sources += files('ipn3ke_ethdev.c',
-	'ipn3ke_representor.c',
-	'ipn3ke_tm.c',
-	'ipn3ke_flow.c')
-deps += ['bus_ifpga', 'sched']
+dep = dependency('libfdt', required: false)
+if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
+	build = false
+	reason = 'missing dependency, "libfdt"'
+endif
+
+if build
+	allow_experimental_apis = true
+
+	sources += files('ipn3ke_ethdev.c',
+		'ipn3ke_representor.c',
+		'ipn3ke_tm.c',
+		'ipn3ke_flow.c')
+	deps += ['bus_ifpga', 'sched', 'pmd_i40e', 'rawdev', 'rawdev_ifpga']
+endif
diff --git a/drivers/raw/ifpga/base/meson.build b/drivers/raw/ifpga/base/meson.build
index 69f6598..b13e13e 100644
--- a/drivers/raw/ifpga/base/meson.build
+++ b/drivers/raw/ifpga/base/meson.build
@@ -25,5 +25,5 @@ sources = [
 
 base_lib = static_library('ifpga_rawdev_base', sources,
 	dependencies: static_rte_eal,
-	c_args: c_args)
+	c_args: cflags)
 base_objs = base_lib.extract_all_objects()
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
index 0ab6fd7..69debff 100644
--- a/drivers/raw/ifpga/meson.build
+++ b/drivers/raw/ifpga/meson.build
@@ -3,18 +3,27 @@
 
 version = 1
 
-subdir('base')
-objs = [base_objs]
-
 dep = dependency('libfdt', required: false)
 if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
 	build = false
 	reason = 'missing dependency, "libfdt"'
 endif
-deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
-	'bus_vdev', 'bus_ifpga', 'net']
-sources = files('ifpga_rawdev.c')
 
-includes += include_directories('base')
+if build
+	subdir('base')
+	objs = [base_objs]
+
+	deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
+		'bus_vdev', 'bus_ifpga', 'net']
+	ext_deps += dep
 
-allow_experimental_apis = true
+	sources = files('ifpga_rawdev.c')
+
+	includes += include_directories('base')
+	includes += include_directories('../../net/ipn3ke')
+
+	allow_experimental_apis = true
+endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 683e3a4..a278552 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -330,7 +330,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 06/19] raw/ifpga/base: align the send buffer for SPI

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 07/19] raw/ifpga/base: add sensor support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 08/19] raw/ifpga/base: introducing sensor APIs

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 09/19] raw/ifpga/base: update SEU register definition

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 10/19] raw/ifpga: add SEU error handler

[Rosen Xu]

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 241 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 241 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 1825143..977dfcf 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -27,6 +27,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -605,6 +607,232 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr)
+		fme_err_read_seu_emr(mgr);
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err)
+		fme_err_read_seu_emr(mgr);
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -652,6 +880,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -677,6 +906,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -696,6 +929,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -720,6 +954,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 11/19] raw/ifpga: add PCIe BDF devices tree scan

[Rosen Xu]

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 551 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 562 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 977dfcf..9834f2a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_malloc.h>
@@ -17,7 +19,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -25,6 +27,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -37,6 +40,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -64,6 +73,494 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(
+		struct ifpga_rawdev *ifpga_rdev, int force_disable);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+		int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
+					sensor->name, value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(
+	struct ifpga_rawdev *ifpga_rdev, int force_disable)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable || force_disable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev,
+					gsd_enable);
+				if (ret == 1 && !gsd_enable) {
+					gsd_enable = 1;
+					i = -1;
+				}
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -372,8 +869,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -838,6 +1336,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -851,7 +1350,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -865,6 +1364,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -983,6 +1490,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1014,13 +1522,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1034,7 +1561,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1061,6 +1589,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 12/19] net/ipn3ke: remove configuration for i40e port bonding

[Rosen Xu]

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/ipn3ke/Makefile                   |   1 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 292 ++++----------------------
 drivers/net/ipn3ke/ipn3ke_flow.c              |   6 +
 drivers/net/ipn3ke/ipn3ke_rawdev_api.h        |  12 ++
 drivers/net/ipn3ke/ipn3ke_representor.c       |  10 +-
 drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map |   6 +
 drivers/raw/ifpga/Makefile                    |   5 +
 drivers/raw/ifpga/ifpga_rawdev.c              |   4 +
 drivers/raw/ifpga/meson.build                 |   5 +-
 9 files changed, 85 insertions(+), 256 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..8c58d71 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,7 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index af87fda..5b5510f 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -35,6 +36,8 @@
 	{ 0, 0 /* sentinel */ },
 };
 
+struct ipn3ke_pub_func ipn3ke_bridge_func;
+
 static int
 ipn3ke_indirect_read(struct ipn3ke_hw *hw, uint32_t *rd_data,
 	uint32_t addr, uint32_t dev_sel, uint32_t eth_group_sel)
@@ -324,7 +327,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -342,6 +346,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -409,9 +419,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -462,7 +469,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -489,7 +500,14 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	if (ipn3ke_bridge_func.get_ifpga_rawdev == NULL)
+		return -ENOMEM;
+	ifpga_dev = ipn3ke_bridge_func.get_ifpga_rawdev(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -501,6 +519,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -508,6 +542,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -553,253 +588,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_flow.c b/drivers/net/ipn3ke/ipn3ke_flow.c
index 9fc3c8b..f857e64 100644
--- a/drivers/net/ipn3ke/ipn3ke_flow.c
+++ b/drivers/net/ipn3ke/ipn3ke_flow.c
@@ -18,6 +18,12 @@
 #include <rte_malloc.h>
 #include <rte_eth_ctrl.h>
 #include <rte_tailq.h>
+#include <rte_rawdev.h>
+#include <rte_rawdev_pmd.h>
+#include <rte_bus_ifpga.h>
+#include <ifpga_common.h>
+#include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
diff --git a/drivers/net/ipn3ke/ipn3ke_rawdev_api.h b/drivers/net/ipn3ke/ipn3ke_rawdev_api.h
index 671fae8..fd2393f 100644
--- a/drivers/net/ipn3ke/ipn3ke_rawdev_api.h
+++ b/drivers/net/ipn3ke/ipn3ke_rawdev_api.h
@@ -59,4 +59,16 @@ struct ifpga_rawdevg_link_info {
 	enum ifpga_rawdev_link_speed link_speed;
 };
 
+struct ipn3ke_pub_func {
+	struct ifpga_rawdev *(*get_ifpga_rawdev)(const struct rte_rawdev *rdv);
+	int (*set_i40e_sw_dev)(uint16_t port_id, struct rte_eth_dev *sw_dev);
+};
+
+/**
+ * @internal
+ * The publid functions of bridge PAC N3000 FPGA and I40e.
+ */
+extern struct ipn3ke_pub_func ipn3ke_bridge_func;
+
+
 #endif /* _IFPGA_RAWDEV_H_ */
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index d37f5e2..8d9ebef 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -2914,12 +2914,18 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	if (representor_param->port_id >= representor_param->hw->port_num)
 		return -ENODEV;
 
+	if (ipn3ke_bridge_func.set_i40e_sw_dev == NULL)
+		return -ENOMEM;
+
 	rpst->ethdev = ethdev;
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		ipn3ke_bridge_func.set_i40e_sw_dev(rpst->i40e_pf_eth_port_id,
+					    rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
diff --git a/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map b/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
index fc8c95e..fbb74ee 100644
--- a/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
+++ b/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
@@ -2,3 +2,9 @@ DPDK_19.05 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	ipn3ke_bridge_func;
+} DPDK_19.05;
diff --git a/drivers/raw/ifpga/Makefile b/drivers/raw/ifpga/Makefile
index 655b292..8fcdb09 100644
--- a/drivers/raw/ifpga/Makefile
+++ b/drivers/raw/ifpga/Makefile
@@ -13,6 +13,7 @@ CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
 CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga_rawdev
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 CFLAGS += -I$(RTE_SDK)/drivers/net/ipn3ke
 LDLIBS += -lrte_eal
 LDLIBS += -lrte_rawdev
@@ -20,6 +21,10 @@ LDLIBS += -lrte_bus_vdev
 LDLIBS += -lrte_kvargs
 LDLIBS += -lrte_bus_pci
 LDLIBS += -lrte_bus_ifpga
+LDLIBS += -lpthread
+LDLIBS += -lfdt
+LDLIBS += -lrte_pmd_i40e
+LDLIBS += -lrte_pmd_ipn3ke
 
 EXPORT_MAP := rte_rawdev_ifpga_version.map
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 9834f2a..6e2242c 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -28,6 +28,7 @@
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
 #include <rte_string_fns.h>
+#include <rte_pmd_i40e.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -1364,6 +1365,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	ipn3ke_bridge_func.get_ifpga_rawdev = ifpga_rawdev_get;
+	ipn3ke_bridge_func.set_i40e_sw_dev = rte_pmd_i40e_set_switch_dev;
+
 	dev = ifpga_rawdev_allocate(rawdev);
 	if (dev == NULL) {
 		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
index 69debff..c3459e2 100644
--- a/drivers/raw/ifpga/meson.build
+++ b/drivers/raw/ifpga/meson.build
@@ -16,14 +16,15 @@ if build
 	subdir('base')
 	objs = [base_objs]
 
-	deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
-		'bus_vdev', 'bus_ifpga', 'net']
+	deps += ['ethdev', 'rawdev', 'pci', 'bus_pci', 'kvargs',
+		'bus_vdev', 'bus_ifpga', 'net', 'i40e', 'ipn3ke']
 	ext_deps += dep
 
 	sources = files('ifpga_rawdev.c')
 
 	includes += include_directories('base')
 	includes += include_directories('../../net/ipn3ke')
+	includes += include_directories('../../net/i40e')
 
 	allow_experimental_apis = true
 endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 13/19] raw/ifpga/base: add secure support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 137 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 ++++++++++++-----
 4 files changed, 198 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..748ab56 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,63 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager")) {
+			sysmgr_offset = offset;
+			break;
+		}
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +576,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 14/19] raw/ifpga/base: configure FEC mode

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 15/19] raw/ifpga/base: clean fme errors

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occurred.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 5d6d630..5905eac 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 6e2242c..e7b1738 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1175,6 +1175,25 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1184,6 +1203,9 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 16/19] raw/ifpga/base: add new API get board info

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..9f0147d 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disaggregate;
+	u8 lightweight;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..799d67d 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweight = id.lightweiht;
+	fme->board_info.disaggregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweight:%s\n"
+			"disaggregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweight),
+			check_support(fme->board_info.disaggregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 17/19] raw/ifpga: add lightweight fpga image support

[Rosen Xu]

From: Andy Pei <andy.pei@intel.com>

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index e7b1738..193ac47 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -836,6 +836,8 @@ static int set_surprise_link_check_aer(
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -862,22 +864,40 @@ static int set_surprise_link_check_aer(
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweight) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
-		uuid.b + 8, sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 18/19] raw/ifpga/base: add multiple cards support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, there is one MAX10 chip in each card, and
all of the sensors are connected to MAX10 chip. To support multiple
cards in one server, we introducing a sensor device list under
intel_max10_device instead of a global list. On the other hand, we
using separate intel_max10_device instance for each opae_adatper.

Add mutex lock on do_transaction() function for SPI driver to avoid
race condition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c            |  40 +++++++---
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          |  14 ++--
 drivers/raw/ifpga/base/opae_hw_api.h          |  15 ++--
 drivers/raw/ifpga/base/opae_i2c.c             |  44 ++++++++---
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_intel_max10.c     | 110 ++++++++++++++------------
 drivers/raw/ifpga/base/opae_intel_max10.h     |  19 +++--
 drivers/raw/ifpga/base/opae_spi.c             |   5 --
 drivers/raw/ifpga/base/opae_spi.h             |   3 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  44 +++++++++--
 drivers/raw/ifpga/ifpga_rawdev.c              |  14 ++--
 12 files changed, 205 insertions(+), 109 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 799d67d..c31a94c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -839,8 +839,13 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	struct ifpga_hw *hw;
 	u32 val;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
@@ -854,7 +859,10 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	fme->board_info.seu = id.seu;
 	fme->board_info.ptp = id.ptp;
 
-	dev_info(fme, "found: board: %s type: %s\n",
+	dev_info(fme, "found: PCI dev: %02x:%02x:%x board: %s type: %s\n",
+			hw->pci_data->bus,
+			hw->pci_data->devid,
+			hw->pci_data->function,
 			board_major_to_string(fme->board_info.major),
 			board_type_to_string(fme->board_info.type));
 
@@ -882,11 +890,11 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
-	if (max10_sys_read(MAX10_BUILD_VER, &val))
+	if (max10_sys_read(fme->max10_dev, MAX10_BUILD_VER, &val))
 		return -EINVAL;
 	fme->board_info.max10_version = val & 0xffffff;
 
-	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+	if (max10_sys_read(fme->max10_dev, NIOS2_FW_VERSION, &val))
 		return -EINVAL;
 	fme->board_info.nios_fw_version = val & 0xffffff;
 
@@ -897,12 +905,12 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	return 0;
 }
 
-static int spi_self_checking(void)
+static int spi_self_checking(struct intel_max10_device *dev)
 {
 	u32 val;
 	int ret;
 
-	ret = max10_sys_read(MAX10_TEST_REG, &val);
+	ret = max10_sys_read(dev, MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
@@ -937,10 +945,11 @@ static int fme_spi_init(struct ifpga_feature *feature)
 		goto spi_fail;
 	}
 
+
 	fme->max10_dev = max10;
 
 	/* SPI self test */
-	if (spi_self_checking()) {
+	if (spi_self_checking(max10)) {
 		ret = -EIO;
 		goto max10_fail;
 	}
@@ -1041,8 +1050,18 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 	struct ifpga_fme_hw *fme = (struct ifpga_fme_hw *)feature->parent;
 	struct altera_spi_device *spi_master;
 	struct intel_max10_device *max10;
+	struct ifpga_hw *hw;
+	struct opae_manager *mgr;
 	int ret = 0;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
+	mgr = hw->adapter->mgr;
+	if (!mgr)
+		return -ENODEV;
+
 	dev_info(fme, "FME SPI Master (NIOS) Init.\n");
 	dev_debug(fme, "FME SPI base addr %p.\n",
 			feature->addr);
@@ -1080,12 +1099,15 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 		goto release_dev;
 	}
 
+	max10->bus = hw->pci_data->bus;
+
 	fme_get_board_interface(fme);
 
 	fme->max10_dev = max10;
+	mgr->sensor_list = &max10->opae_sensor_list;
 
 	/* SPI self test */
-	if (spi_self_checking())
+	if (spi_self_checking(max10))
 		goto spi_fail;
 
 	return ret;
@@ -1344,7 +1366,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(dev, PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1372,7 +1394,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(sensor->value_reg, value)) {
+	if (max10_sys_read(dev, sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_debug.c b/drivers/raw/ifpga/base/opae_debug.c
index 88f2d5c..dad3ea3 100644
--- a/drivers/raw/ifpga/base/opae_debug.c
+++ b/drivers/raw/ifpga/base/opae_debug.c
@@ -59,6 +59,9 @@ static void opae_adapter_data_dump(void *data)
 		opae_log("OPAE Adapter Type = PCI\n");
 		opae_log("PCI Device ID: 0x%04x\n", d_pci->device_id);
 		opae_log("PCI Vendor ID: 0x%04x\n", d_pci->vendor_id);
+		opae_log("PCI bus: 0x%04x\n", d_pci->bus);
+		opae_log("PCI devid: 0x%04x\n", d_pci->devid);
+		opae_log("PCI function: 0x%04x\n", d_pci->function);
 
 		for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 			r = &d_pci->region[i];
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 1ccc967..c969dfe 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -583,11 +583,12 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_id(unsigned int id)
+opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (sensor->id == id)
 			return sensor;
 
@@ -601,11 +602,12 @@ struct opae_sensor_info *
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_name(const char *name)
+opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (!strcmp(sensor->name, name))
 			return sensor;
 
@@ -630,7 +632,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_name(name);
+	sensor = opae_mgr_get_sensor_by_name(mgr, name);
 	if (!sensor)
 		return -ENODEV;
 
@@ -658,7 +660,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_id(id);
+	sensor = opae_mgr_get_sensor_by_id(mgr, id);
 	if (!sensor)
 		return -ENODEV;
 
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index b78fbd5..cdf369f 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -40,6 +40,7 @@ struct opae_manager {
 	struct opae_adapter *adapter;
 	struct opae_manager_ops *ops;
 	struct opae_manager_networking_ops *network_ops;
+	struct opae_sensor_list *sensor_list;
 	void *data;
 };
 
@@ -75,9 +76,8 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
-extern struct opae_sensor_list opae_sensor_list;
-#define opae_mgr_for_each_sensor(sensor) \
-	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+#define opae_mgr_for_each_sensor(mgr, sensor) \
+	TAILQ_FOREACH(sensor, mgr->sensor_list, node)
 
 /* OPAE Manager APIs */
 struct opae_manager *
@@ -88,8 +88,10 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
-struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
-struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id);
 int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
 		const char *name, unsigned int *value);
 int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
@@ -241,6 +243,9 @@ struct opae_adapter_data_pci {
 	enum opae_adapter_type type;
 	u16 device_id;
 	u16 vendor_id;
+	u16 bus; /*Device bus for PCI */
+	u16 devid; /* Device ID */
+	u16 function; /* Device function */
 	struct opae_reg_region region[PCI_MAX_RESOURCE];
 	int vfio_dev_fd;  /* VFIO device file descriptor */
 };
diff --git a/drivers/raw/ifpga/base/opae_i2c.c b/drivers/raw/ifpga/base/opae_i2c.c
index f8bc247..846d751 100644
--- a/drivers/raw/ifpga/base/opae_i2c.c
+++ b/drivers/raw/ifpga/base/opae_i2c.c
@@ -28,6 +28,9 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 {
 	u8 msgbuf[2];
 	int i = 0;
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
 
 	if (flags & I2C_FLAG_ADDR16)
 		msgbuf[i++] = offset >> 8;
@@ -49,10 +52,16 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		},
 	};
 
-	if (!dev->xfer)
-		return -ENODEV;
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	ret = i2c_transfer(dev, msg, 2);
 
-	return i2c_transfer(dev, msg, 2);
+exit:
+	pthread_mutex_unlock(&dev->lock);
+	return ret;
 }
 
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
@@ -63,12 +72,18 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	int ret;
 	int i = 0;
 
-	if (!dev->xfer)
-		return -ENODEV;
+	pthread_mutex_lock(&dev->lock);
+
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
 
 	buf = opae_malloc(I2C_MAX_OFFSET_LEN + len);
-	if (!buf)
-		return -ENOMEM;
+	if (!buf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
 
 	msg.addr = slave_addr;
 	msg.flags = 0;
@@ -84,6 +99,8 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	ret = i2c_transfer(dev, &msg, 1);
 
 	opae_free(buf);
+exit:
+	pthread_mutex_unlock(&dev->lock);
 	return ret;
 }
 
@@ -477,14 +494,19 @@ struct altera_i2c_dev *altera_i2c_probe(void *base)
 	dev->i2c_clk = dev->i2c_param.ref_clk * 1000000;
 	dev->xfer = altera_i2c_xfer;
 
+	if (pthread_mutex_init(&dev->lock, NULL))
+		return NULL;
+
 	altera_i2c_hardware_init(dev);
 
 	return dev;
 }
 
-int altera_i2c_remove(struct altera_i2c_dev *dev)
+void altera_i2c_remove(struct altera_i2c_dev *dev)
 {
-	altera_i2c_disable(dev);
-
-	return 0;
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
+		altera_i2c_disable(dev);
+		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/base/opae_i2c.h b/drivers/raw/ifpga/base/opae_i2c.h
index 8890c8f..266e127 100644
--- a/drivers/raw/ifpga/base/opae_i2c.h
+++ b/drivers/raw/ifpga/base/opae_i2c.h
@@ -93,6 +93,7 @@ struct altera_i2c_dev {
 	u32 isr_mask;
 	u8 *buf;
 	int (*xfer)(struct altera_i2c_dev *dev, struct i2c_msg *msg, int num);
+	pthread_mutex_t lock;
 };
 
 /**
@@ -114,7 +115,7 @@ enum i2c_msg_flags {
 };
 
 struct altera_i2c_dev *altera_i2c_probe(void *base);
-int altera_i2c_remove(struct altera_i2c_dev *dev);
+void altera_i2c_remove(struct altera_i2c_dev *dev);
 int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		u32 offset, u8 *buf, u32 count);
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 748ab56..8e23ca1 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -5,45 +5,50 @@
 #include "opae_intel_max10.h"
 #include <libfdt.h>
 
-static struct intel_max10_device *g_max10;
-
-struct opae_sensor_list opae_sensor_list =
-	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
-
-int max10_reg_read(unsigned int reg, unsigned int *val)
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_read(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x\n", __func__, dev->bus, reg);
+
+	return spi_transaction_read(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)val);
 }
 
-int max10_reg_write(unsigned int reg, unsigned int val)
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val)
 {
 	unsigned int tmp = val;
 
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_write(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x, val:0x%x\n", __func__,
+			dev->bus, reg, val);
+
+	return spi_transaction_write(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)&tmp);
 }
 
-int max10_sys_read(unsigned int offset, unsigned int *val)
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_read(g_max10->base + offset, val);
+
+	return max10_reg_read(dev, dev->base + offset, val);
 }
 
-int max10_sys_write(unsigned int offset, unsigned int val)
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_write(g_max10->base + offset, val);
+	return max10_reg_write(dev, dev->base + offset, val);
 }
 
 static struct max10_compatible_id max10_id_table[] = {
@@ -86,8 +91,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
-static int altera_nor_flash_read(u32 offset,
-		void *buffer, u32 len)
+static int altera_nor_flash_read(struct intel_max10_device *dev,
+	u32 offset, void *buffer, u32 len)
 {
 	int word_len;
 	int i;
@@ -95,13 +100,13 @@ static int altera_nor_flash_read(u32 offset,
 	unsigned int value;
 	int ret;
 
-	if (!buffer || len <= 0)
+	if (!dev || !buffer || len <= 0)
 		return -ENODEV;
 
 	word_len = len/4;
 
 	for (i = 0; i < word_len; i++) {
-		ret = max10_reg_read(offset + i*4,
+		ret = max10_reg_read(dev, offset + i*4,
 				&value);
 		if (ret)
 			return -EBUSY;
@@ -112,12 +117,12 @@ static int altera_nor_flash_read(u32 offset,
 	return 0;
 }
 
-static int enable_nor_flash(bool on)
+static int enable_nor_flash(struct intel_max10_device *dev, bool on)
 {
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_sys_read(RSU_REG, &val);
+	ret = max10_sys_read(dev, RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -128,7 +133,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_sys_write(RSU_REG, val);
+	return max10_sys_write(dev, RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -140,7 +145,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_sys_read(DT_AVAIL_REG, &val);
+	ret = max10_sys_read(max10, DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -151,19 +156,19 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
+	ret = max10_sys_read(max10, DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
 	}
 
-	ret = enable_nor_flash(true);
+	ret = enable_nor_flash(max10, true);
 	if (ret) {
 		dev_err(max10 "fail to enable flash\n");
 		return ret;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	ret = altera_nor_flash_read(max10, dt_addr, &hdr, sizeof(hdr));
 	if (ret) {
 		dev_err(max10 "read fdt header fail\n");
 		goto done;
@@ -188,7 +193,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		goto done;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	ret = altera_nor_flash_read(max10, dt_addr, fdt_root, dt_size);
 	if (ret) {
 		dev_err(max10 "cannot read device table\n");
 		goto done;
@@ -207,7 +212,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	max10->fdt_root = fdt_root;
 
 done:
-	ret = enable_nor_flash(false);
+	ret = enable_nor_flash(max10, false);
 
 	if (ret && fdt_root)
 		opae_free(fdt_root);
@@ -298,12 +303,12 @@ static int fdt_get_named_reg(const void *fdt, int node, const char *name,
 	return fdt_get_reg(fdt, node, idx, start, size);
 }
 
-static void max10_sensor_uinit(void)
+static void max10_sensor_uinit(struct intel_max10_device *dev)
 {
 	struct opae_sensor_info *info;
 
-	TAILQ_FOREACH(info, &opae_sensor_list, node) {
-		TAILQ_REMOVE(&opae_sensor_list, info, node);
+	TAILQ_FOREACH(info, &dev->opae_sensor_list, node) {
+		TAILQ_REMOVE(&dev->opae_sensor_list, info, node);
 		opae_free(info);
 	}
 }
@@ -313,8 +318,8 @@ static bool sensor_reg_valid(struct sensor_reg *reg)
 	return !!reg->size;
 }
 
-static int max10_add_sensor(struct raw_sensor_info *info,
-		struct opae_sensor_info *sensor)
+static int max10_add_sensor(struct intel_max10_device *dev,
+	struct raw_sensor_info *info, struct opae_sensor_info *sensor)
 {
 	int i;
 	int ret = 0;
@@ -332,12 +337,15 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_sys_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(dev, info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
-		if (val == 0xdeadbeef)
+		if (val == 0xdeadbeef) {
+			dev_debug(dev, "%s: sensor:%s invalid 0x%x at:%d\n",
+				__func__, sensor->name, val, i);
 			continue;
+		}
 
 		val *= info->multiplier;
 
@@ -458,7 +466,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
 		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
 
-		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+		dev_debug(dev, "found sensor from DTB: %s: %s: %u: %u\n",
 				raw->name, raw->type,
 				raw->id, raw->multiplier);
 
@@ -473,15 +481,16 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 			goto free_sensor;
 		}
 
-		if (max10_add_sensor(raw, sensor)) {
+		if (max10_add_sensor(dev, raw, sensor)) {
 			ret = -EINVAL;
 			opae_free(sensor);
 			goto free_sensor;
 		}
 
-		if (sensor->flags & OPAE_SENSOR_VALID)
-			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
-		else
+		if (sensor->flags & OPAE_SENSOR_VALID) {
+			TAILQ_INSERT_TAIL(&dev->opae_sensor_list, sensor, node);
+			dev_info(dev, "found valid sensor: %s\n", sensor->name);
+		} else
 			opae_free(sensor);
 
 		opae_free(raw);
@@ -492,7 +501,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 free_sensor:
 	if (raw)
 		opae_free(raw);
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 	return ret;
 }
 
@@ -500,7 +509,7 @@ static int check_max10_version(struct intel_max10_device *dev)
 {
 	unsigned int v;
 
-	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+	if (!max10_reg_read(dev, MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
 				&v)) {
 		if (v != 0xffffffff) {
 			dev_info(dev, "secure MAX10 detected\n");
@@ -565,6 +574,8 @@ struct intel_max10_device *
 	if (!dev)
 		return NULL;
 
+	TAILQ_INIT(&dev->opae_sensor_list);
+
 	dev->spi_master = spi;
 
 	dev->spi_tran_dev = spi_transaction_init(spi, chipselect);
@@ -573,9 +584,6 @@ struct intel_max10_device *
 		goto free_dev;
 	}
 
-	/* set the max10 device firstly */
-	g_max10 = dev;
-
 	/* check the max10 version */
 	ret = check_max10_version(dev);
 	if (ret) {
@@ -601,7 +609,7 @@ struct intel_max10_device *
 	}
 
 	/* read FPGA loading information */
-	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
+	ret = max10_sys_read(dev, FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
 		goto release_max10_hw;
@@ -611,14 +619,13 @@ struct intel_max10_device *
 	return dev;
 
 release_max10_hw:
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return NULL;
@@ -629,7 +636,7 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
@@ -637,7 +644,6 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index e632941..123cdc4 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -26,6 +26,9 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SECURE		BIT(6)
 #define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
@@ -33,6 +36,8 @@ struct intel_max10_device {
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
 	unsigned int base; /* max10 base address */
+	u16 bus;
+	struct opae_sensor_list opae_sensor_list;
 };
 
 /* retimer speed */
@@ -136,17 +141,19 @@ struct opae_retimer_status {
 
 #define DFT_MAX_SIZE		0x7e0000
 
-int max10_reg_read(unsigned int reg, unsigned int *val);
-int max10_reg_write(unsigned int reg, unsigned int val);
-int max10_sys_read(unsigned int offset, unsigned int *val);
-int max10_sys_write(unsigned int offset, unsigned int val);
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val);
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val);
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val);
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
-/** List of opae sensors */
-TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
 
 #define SENSOR_REG_VALUE 0x0
 #define SENSOR_REG_HIGH_WARN 0x1
diff --git a/drivers/raw/ifpga/base/opae_spi.c b/drivers/raw/ifpga/base/opae_spi.c
index cc52782..bfdc83e 100644
--- a/drivers/raw/ifpga/base/opae_spi.c
+++ b/drivers/raw/ifpga/base/opae_spi.c
@@ -181,7 +181,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 	u32 rxd;
 	unsigned int tx_data;
 	u32 status;
-	int retry = 0;
 	int ret;
 
 	while (count < dev->len) {
@@ -194,10 +193,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 				return -EIO;
 			if (status & ALTERA_SPI_STATUS_RRDY_MSK)
 				break;
-			if (retry++ > SPI_MAX_RETRY) {
-				dev_err(dev, "%s, read timeout\n", __func__);
-				return -EBUSY;
-			}
 		}
 
 		ret = spi_reg_read(dev, ALTERA_SPI_RXDATA, &rxd);
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index 6355deb..d20a4c3 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -38,7 +38,7 @@
 #define SPI_WRITE 0x20
 #define WRITE_DATA_MASK GENMASK_ULL(31, 0)
 
-#define SPI_MAX_RETRY 100000
+#define SPI_MAX_RETRY 1000000
 
 #define TYPE_SPI 0
 #define TYPE_NIOS_SPI 1
@@ -102,6 +102,7 @@ struct spi_transaction_dev {
 	struct altera_spi_device *dev;
 	int chipselect;
 	struct spi_tran_buffer *buffer;
+	pthread_mutex_t lock;
 };
 
 struct spi_tran_header {
diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 06ca625..013efee 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -154,6 +154,8 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	unsigned int rx_len = 0;
 	int retry = 0;
 	int spi_flags;
+	unsigned long timeout = msecs_to_timer_cycles(1000);
+	unsigned long ticks;
 	unsigned int resp_max_len = 2 * resp_len;
 
 	print_buffer("before bytes:", send_data, send_len);
@@ -207,8 +209,11 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	if (ret != SPI_FOUND_EOP) {
 		tx_buffer = NULL;
 		tx_len = 0;
-		if (retry++ > 10) {
-			dev_err(NULL, "cannot found valid data from SPI\n");
+		ticks = rte_get_timer_cycles();
+		if (time_after(ticks, timeout) &&
+				retry++ > SPI_MAX_RETRY) {
+			dev_err(NULL, "Have retry %d, found invalid packet data\n",
+				retry);
 			return -EBUSY;
 		}
 
@@ -427,23 +432,36 @@ static int do_transaction(struct spi_transaction_dev *dev, unsigned int addr,
 int spi_transaction_read(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_READ : SPI_TRAN_NON_SEQ_READ);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 int spi_transaction_write(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_WRITE : SPI_TRAN_NON_SEQ_WRITE);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 		int chipselect)
 {
 	struct spi_transaction_dev *spi_tran_dev;
+	int ret;
 
 	spi_tran_dev = opae_malloc(sizeof(struct spi_transaction_dev));
 	if (!spi_tran_dev)
@@ -453,18 +471,28 @@ struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 	spi_tran_dev->chipselect = chipselect;
 
 	spi_tran_dev->buffer = opae_malloc(sizeof(struct spi_tran_buffer));
-	if (!spi_tran_dev->buffer) {
-		opae_free(spi_tran_dev);
-		return NULL;
+	if (!spi_tran_dev->buffer)
+		goto err;
+
+	ret = pthread_mutex_init(&spi_tran_dev->lock, NULL);
+	if (ret) {
+		dev_err(spi_tran_dev, "fail to init mutex lock\n");
+		goto err;
 	}
 
 	return spi_tran_dev;
+
+err:
+	opae_free(spi_tran_dev);
+	return NULL;
 }
 
 void spi_transaction_remove(struct spi_transaction_dev *dev)
 {
 	if (dev && dev->buffer)
 		opae_free(dev->buffer);
-	if (dev)
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
 		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 193ac47..fb5d9b1 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -361,7 +361,7 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 	if (!mgr)
 		return -ENODEV;
 
-	opae_mgr_for_each_sensor(sensor) {
+	opae_mgr_for_each_sensor(mgr, sensor) {
 		if (!(sensor->flags & OPAE_SENSOR_VALID))
 			goto fail;
 
@@ -370,8 +370,8 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 			goto fail;
 
 		if (value == 0xdeadbeef) {
-			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
-					sensor->name, value);
+			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value %x\n",
+					raw_dev->dev_id, sensor->name, value);
 			continue;
 		}
 
@@ -394,8 +394,9 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 		/* monitor 12V AUX sensor */
 		if (!strcmp(sensor->name, "12V AUX Voltage")) {
 			if (value < AUX_VOLTAGE_WARN) {
-				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
-						sensor->name, value);
+				IFPGA_RAWDEV_PMD_INFO(
+					"%s reach theshold %d mV\n",
+					sensor->name, value);
 				*gsd_start = true;
 				break;
 			}
@@ -1433,6 +1434,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->bus = pci_dev->addr.bus;
+	data->devid = pci_dev->addr.devid;
+	data->function = pci_dev->addr.function;
 	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
-- 
1.8.3.1

[dpdk-dev] [PATCH v17 19/19] raw/ifpga: introducing new irq API

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing new register and unregister API for ifpga interrupt.
1. register FME and AFU interrupt
   ifpga_register_msix_irq()

2. unregister FME and AFU interrupt
   ifpga_unregister_msix_irq()

On PAC N3000 card, there is one PCIe MSIX interrupt for FME managerment,
like the error report, thermal management, we use this interrupt in
ifpga_rawdev device driver. on the other hand, there are about 4 PCIe
MSIX interrupts are reserved for AFU which end-user can use those
interrupts in their AFU logic design. End-user can use those APIs to
register interrupt handler in their AFU drivers.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 105 +++++++++++++++++++++++++++------------
 drivers/raw/ifpga/ifpga_rawdev.h |  14 ++++++
 2 files changed, 88 insertions(+), 31 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index fb5d9b1..a79b581 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -79,6 +79,10 @@
 static int ifpga_monitor_start;
 static pthread_t ifpga_monitor_start_thread;
 
+#define IFPGA_MAX_IRQ 12
+/* 0 for FME interrupt, others are reserved for AFU irq */
+static struct rte_intr_handle ifpga_irq_handle[IFPGA_MAX_IRQ];
+
 static struct ifpga_rawdev *
 ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
 static int set_surprise_link_check_aer(
@@ -1328,53 +1332,90 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 	fme_err_handle_catfatal_error(mgr);
 }
 
-static struct rte_intr_handle fme_intr_handle;
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg)
+{
+	struct rte_intr_handle intr_handle;
+
+	if (type == IFPGA_FME_IRQ)
+		intr_handle = ifpga_irq_handle[0];
+	else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+	rte_intr_efd_disable(&intr_handle);
+
+	return rte_intr_callback_unregister(&intr_handle,
+			handler, arg);
+}
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg)
 {
 	int ret;
-	struct fpga_fme_err_irq_set err_irq_set;
+	struct rte_intr_handle intr_handle;
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_accelerator *acc;
 
-	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+	adapter = ifpga_rawdev_get_priv(dev);
+	if (!adapter)
+		return -ENODEV;
 
-	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
-	if (ret)
-		return -EINVAL;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
 
-	fme_intr_handle.fd = fme_intr_handle.efds[0];
+	if (type == IFPGA_FME_IRQ) {
+		intr_handle = ifpga_irq_handle[0];
+		count = 1;
+	} else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
-			fme_intr_handle.vfio_dev_fd,
-			fme_intr_handle.efds[0], fme_intr_handle.fd);
+	intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
 
-	err_irq_set.evtfd = fme_intr_handle.efds[0];
-	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	ret = rte_intr_efd_enable(&intr_handle, count);
 	if (ret)
-		return -EINVAL;
+		return -ENODEV;
+
+	intr_handle.fd = intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("register %s irq, vfio_fd=%d, fd=%d\n",
+			name, intr_handle.vfio_dev_fd,
+			intr_handle.fd);
+
+	if (type == IFPGA_FME_IRQ) {
+		struct fpga_fme_err_irq_set err_irq_set;
+		err_irq_set.evtfd = intr_handle.efds[0];
+
+		ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+		if (ret)
+			return -EINVAL;
+	} else if (type == IFPGA_AFU_IRQ) {
+		acc = opae_adapter_get_acc(adapter, port_id);
+		if (!acc)
+			return -EINVAL;
 
-	/* register FME interrupt using DPDK API */
-	ret = rte_intr_callback_register(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
+		ret = opae_acc_set_irq(acc, vec_start, count, intr_handle.efds);
+		if (ret)
+			return -EINVAL;
+	}
+
+	/* register interrupt handler using DPDK API */
+	ret = rte_intr_callback_register(&intr_handle,
+			handler, (void *)arg);
 	if (ret)
 		return -EINVAL;
 
-	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+	IFPGA_RAWDEV_PMD_INFO("success register %s interrupt\n", name);
 
 	return 0;
 }
 
 static int
-ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
-{
-	rte_intr_efd_disable(&fme_intr_handle);
-
-	return rte_intr_callback_unregister(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
-}
-
-static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -1463,7 +1504,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
-	ret = ifpga_register_fme_interrupt(mgr);
+	ret = ifpga_register_msix_irq(rawdev, 0, IFPGA_FME_IRQ, 0, 0,
+			fme_interrupt_handler, "fme_irq", mgr);
 	if (ret)
 		goto free_adapter_data;
 
@@ -1515,7 +1557,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	if (!mgr)
 		return -ENODEV;
 
-	if (ifpga_unregister_fme_interrupt(mgr))
+	if (ifpga_unregister_msix_irq(IFPGA_FME_IRQ, 0,
+				fme_interrupt_handler, mgr))
 		return -EINVAL;
 
 	opae_adapter_data_free(adapter->data);
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index bd42083..7754beb 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -62,4 +62,18 @@ struct ifpga_rawdev {
 struct ifpga_rawdev *
 ifpga_rawdev_get(const struct rte_rawdev *rawdev);
 
+enum ifpga_irq_type {
+	IFPGA_FME_IRQ = 0,
+	IFPGA_AFU_IRQ = 1,
+};
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg);
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Rosen Xu]

This patch set adds PCIe AER disable and FPGA interrupt support for
ipn3ke. It also provides a small rework for port bonding between FPGA
line side port and I40e PF port.

What is the PCI Express AER(Advanced Error Reporting)?
Advanced Error Reporting capability is implemented with a PCI Express
advanced error reporting extended capability structure providing more
robust error reporting. It's also one of PCI Express error reporting
paradigms. AER is supported by most of PCIe devices.

In PAC N3000 card, some uncertainty errors will cause FPGA reload,
such as temperature is higher than threshold. From Software point of
view, FPGA reload means FPGA unplug and plug. For avoiding system
crash we need to clear AER register before these errors occur.

Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga
rawdev driver, we implement a FME interrupt function to notify errors
reported by FME. Besides this, OPAE share code also provide a common
AFU interrupt API for users to register their own interrupt functions.

v18 updates:
=========
- remove rte_exit.

v17 updates:
=========
- remove rte_panic.

v16 updates:
=========
- rebase to DPDK 19.11 RC-2.

v15 updates:
=========
- fix share library build issues.

v14 updates:
=========
- fix coding style and remove unnecessary comments.

v13 updates:
=========
- fix meson build issue.

v12 updates:
=========
- fix meson build issue.
 
v11 updates:
=========
- move symbol ifpga_rawdev_ge to the EXPERIMENTAL section of 
  drivers/raw/ifpga/rte_rawdev_ifpga_version.map

v10 updates:
=========
- introducing new irq API
- fix meson build issue

v9 updates:
=========
- Add mutex lock on do_transaction() function for SPI driver to avoid race
  condition.

v8 updates:
=========
- add multiple cards support.

v7 updates:
==========
- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
- function rte_pmd_i40e_set_switch_dev works as an external API,
  use port_id but not rte_eth_dev as parameter.
- add doxygen header here for the new API.
- update the rte_pmd_i40e_version.map.
- fix coding style issue.
- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
- for functions with a lot of similarity, extract out common function to reduce
  duplication.

v6 updates:
=========
- correct author information.
- correct typo in commit message and remove Gerrit Change-Id's before
  submitting upstream

v5 updates:
==========
- add lightweight fpga image support. in lightweight fpga image mode,
  ipn3ke representor will not be probed.

v4 updates:
===========
- align with new naming standard.

v3 updates:
===========
- Add FPGA network side port MTU configuration

v2 updates:
===========
- Add AUX feature support

Andy Pei (2):
  net/i40e: i40e support ipn3ke FPGA port bonding
  raw/ifpga: add lightweight fpga image support

Rosen Xu (3):
  raw/ifpga: add SEU error handler
  raw/ifpga: add PCIe BDF devices tree scan
  net/ipn3ke: remove configuration for i40e port bonding

Tianfei zhang (14):
  raw/ifpga/base: add irq support
  raw/ifpga/base: clear pending bit
  raw/ifpga/base: add SEU error support
  raw/ifpga/base: add device tree support
  raw/ifpga/base: align the send buffer for SPI
  raw/ifpga/base: add sensor support
  raw/ifpga/base: introducing sensor APIs
  raw/ifpga/base: update SEU register definition
  raw/ifpga/base: add secure support
  raw/ifpga/base: configure FEC mode
  raw/ifpga/base: clean fme errors
  raw/ifpga/base: add new API get board info
  raw/ifpga/base: add multiple cards support
  raw/ifpga: introducing new irq API

 config/common_base                            |   4 +-
 config/common_linux                           |   6 +
 drivers/meson.build                           |   6 +-
 drivers/net/i40e/base/i40e_type.h             |   3 +
 drivers/net/i40e/i40e_ethdev.c                |   6 +
 drivers/net/i40e/rte_pmd_i40e.c               |  21 +
 drivers/net/i40e/rte_pmd_i40e.h               |  18 +
 drivers/net/i40e/rte_pmd_i40e_version.map     |   8 +-
 drivers/net/ipn3ke/Makefile                   |   1 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 292 ++-------
 drivers/net/ipn3ke/ipn3ke_flow.c              |   6 +
 drivers/net/ipn3ke/ipn3ke_rawdev_api.h        |  12 +
 drivers/net/ipn3ke/ipn3ke_representor.c       |  10 +-
 drivers/net/ipn3ke/meson.build                |  24 +-
 drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map |   6 +
 drivers/raw/ifpga/Makefile                    |   5 +
 drivers/raw/ifpga/base/ifpga_api.c            |  21 +
 drivers/raw/ifpga/base/ifpga_defines.h        |  75 ++-
 drivers/raw/ifpga/base/ifpga_feature_dev.c    |  59 ++
 drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c            | 166 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c      |  74 ++-
 drivers/raw/ifpga/base/ifpga_hw.h             |   2 +-
 drivers/raw/ifpga/base/ifpga_port.c           |  18 +
 drivers/raw/ifpga/base/ifpga_port_error.c     |  19 +
 drivers/raw/ifpga/base/meson.build            |   2 +-
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          | 137 ++++
 drivers/raw/ifpga/base/opae_hw_api.h          |  26 +
 drivers/raw/ifpga/base/opae_i2c.c             |  44 +-
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
 drivers/raw/ifpga/base/opae_intel_max10.c     | 599 ++++++++++++++++-
 drivers/raw/ifpga/base/opae_intel_max10.h     | 157 ++++-
 drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
 drivers/raw/ifpga/base/opae_spi.c             |   5 -
 drivers/raw/ifpga/base/opae_spi.h             |  26 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  84 ++-
 drivers/raw/ifpga/ifpga_rawdev.c              | 903 +++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h              |  30 +
 drivers/raw/ifpga/meson.build                 |  26 +-
 mk/rte.app.mk                                 |   2 +-
 42 files changed, 2480 insertions(+), 441 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v18 01/19] net/i40e: i40e support ipn3ke FPGA port bonding

[Rosen Xu]

From: Andy Pei <andy.pei@intel.com>

In ipn3ke, each FPGA network side port bonding to an i40e pf,
each i40e pf link status should get data from FPGA network,
side port. This patch provide bonding relationship.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/i40e/base/i40e_type.h         |  3 +++
 drivers/net/i40e/i40e_ethdev.c            |  6 ++++++
 drivers/net/i40e/rte_pmd_i40e.c           | 21 +++++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e.h           | 18 ++++++++++++++++++
 drivers/net/i40e/rte_pmd_i40e_version.map |  8 +++++++-
 5 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/drivers/net/i40e/base/i40e_type.h b/drivers/net/i40e/base/i40e_type.h
index 112866b..06863d7 100644
--- a/drivers/net/i40e/base/i40e_type.h
+++ b/drivers/net/i40e/base/i40e_type.h
@@ -660,6 +660,9 @@ struct i40e_hw {
 	struct i40e_nvm_info nvm;
 	struct i40e_fc_info fc;
 
+	/* switch device is used to get link status when i40e is in ipn3ke */
+	struct rte_eth_dev *switch_dev;
+
 	/* pci info */
 	u16 device_id;
 	u16 vendor_id;
diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c
index 9a81f99..1afddeb 100644
--- a/drivers/net/i40e/i40e_ethdev.c
+++ b/drivers/net/i40e/i40e_ethdev.c
@@ -1393,6 +1393,9 @@ static inline void i40e_config_automask(struct i40e_pf *pf)
 	hw->adapter_stopped = 0;
 	hw->adapter_closed = 0;
 
+	/* Init switch device pointer */
+	hw->switch_dev = NULL;
+
 	/*
 	 * Switch Tag value should not be identical to either the First Tag
 	 * or Second Tag values. So set something other than common Ethertype
@@ -2906,6 +2909,9 @@ void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
 	else
 		update_link_aq(hw, &link, enable_lse, wait_to_complete);
 
+	if (hw->switch_dev)
+		rte_eth_linkstatus_get(hw->switch_dev, &link);
+
 	ret = rte_eth_linkstatus_set(dev, &link);
 	i40e_notify_all_vfs_link_status(dev);
 
diff --git a/drivers/net/i40e/rte_pmd_i40e.c b/drivers/net/i40e/rte_pmd_i40e.c
index 4c3c708..fdcb1a4 100644
--- a/drivers/net/i40e/rte_pmd_i40e.c
+++ b/drivers/net/i40e/rte_pmd_i40e.c
@@ -3207,3 +3207,24 @@ int rte_pmd_i40e_flow_add_del_packet_template(
 	I40E_WRITE_FLUSH(hw);
 	return 0;
 }
+
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev)
+{
+	struct rte_eth_dev *i40e_dev;
+	struct i40e_hw *hw;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+
+	i40e_dev = &rte_eth_devices[port_id];
+	if (!is_i40e_supported(i40e_dev))
+		return -ENOTSUP;
+
+	hw = I40E_DEV_PRIVATE_TO_HW(i40e_dev->data->dev_private);
+	if (!hw)
+		return -1;
+
+	hw->switch_dev = switch_dev;
+
+	return 0;
+}
diff --git a/drivers/net/i40e/rte_pmd_i40e.h b/drivers/net/i40e/rte_pmd_i40e.h
index faac9e2..915cdf0 100644
--- a/drivers/net/i40e/rte_pmd_i40e.h
+++ b/drivers/net/i40e/rte_pmd_i40e.h
@@ -1061,4 +1061,22 @@ int rte_pmd_i40e_inset_set(uint16_t port, uint8_t pctype,
 	return 0;
 }
 
+/**
+ * For ipn3ke, i40e works with FPGA.
+ * In this situation, i40e get link status from fpga,
+ * fpga works as switch_dev for i40e.
+ * This function set switch_dev for i40e.
+ *
+ * @param port_id
+ *    port_id of i40e device to be set switch device.
+ * @param switch_dev
+ *    target switch device from which i40e device to get link status from.
+ * @return
+ *   - (less than 0) if failed.
+ *   - (0) if success.
+ */
+__rte_experimental
+int
+rte_pmd_i40e_set_switch_dev(uint16_t port_id, struct rte_eth_dev *switch_dev);
+
 #endif /* _PMD_I40E_H_ */
diff --git a/drivers/net/i40e/rte_pmd_i40e_version.map b/drivers/net/i40e/rte_pmd_i40e_version.map
index cccd576..ca5bcee 100644
--- a/drivers/net/i40e/rte_pmd_i40e_version.map
+++ b/drivers/net/i40e/rte_pmd_i40e_version.map
@@ -64,4 +64,10 @@ DPDK_18.02 {
 
 	rte_pmd_i40e_inset_get;
 	rte_pmd_i40e_inset_set;
-} DPDK_17.11;
\ No newline at end of file
+} DPDK_17.11;
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_i40e_set_switch_dev;
+};
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 02/19] raw/ifpga/base: add irq support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME global error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

To build this feature, CONFIG_RTE_EAL_VFIO should be enabled.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                         |  2 +-
 config/common_linux                        |  6 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 59 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 19 ++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 18 +++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 19 ++++++++++
 6 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/config/common_base b/config/common_base
index 9142778..36ab13d 100644
--- a/config/common_base
+++ b/config/common_base
@@ -788,7 +788,7 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
 #
 # Compile PMD for Intel FPGA raw device
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
 
 #
 # Compile PMD for Intel IOAT raw device
diff --git a/config/common_linux b/config/common_linux
index 96e2e1f..a78b8c6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -68,3 +68,9 @@ CONFIG_RTE_LIBRTE_HINIC_PMD=y
 # Hisilicon HNS3 PMD driver
 #
 CONFIG_RTE_LIBRTE_HNS3_PMD=y
+
+#
+# Compile PMD for Intel FPGA raw device
+# To compile, CONFIG_RTE_EAL_VFIO should be enabled.
+#
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..0f852a7 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,61 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+#define FPGA_MAX_MSIX_VEC_COUNT	128
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MAX_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	/* 0 < irq_set->count < FPGA_MAX_MSIX_VEC_COUNT */
+	irq_set->count = count ?
+		(count > FPGA_MAX_MSIX_VEC_COUNT ?
+		 FPGA_MAX_MSIX_VEC_COUNT : count) : 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..2978c79 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,28 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&fme->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..c0aaf01 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,27 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..189f762 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,28 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 03/19] raw/ifpga/base: clear pending bit

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 2978c79..be041ec 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 04/19] raw/ifpga/base: add SEU error support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 31 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index be041ec..5d6d630 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,33 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr(struct ifpga_fme_hw *fme,
+		u64 *val, bool high)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+				FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	if (high)
+		*val = readq(&fme_err->seu_emr_h);
+	else
+		*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +297,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr(fme, &prop->data, 0);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr(fme, &prop->data, 1);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 05/19] raw/ifpga/base: add device tree support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

To eliminate build error, we currently do not compile raw/ifpga
and net/ipn3ke. User should install libfdt and libfdt-devel first,
modify config/common_linux, CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
to CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y, modify config/common_base,
CONFIG_RTE_LIBRTE_IPN3KE_PMD=n to CONFIG_RTE_LIBRTE_IPN3KE_PMD=y.
Then this function can work.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 config/common_base                        |   2 +-
 config/common_linux                       |   2 +-
 drivers/meson.build                       |   6 +-
 drivers/net/ipn3ke/meson.build            |  24 +++-
 drivers/raw/ifpga/base/meson.build        |   2 +-
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 drivers/raw/ifpga/meson.build             |  25 ++--
 mk/rte.app.mk                             |   2 +-
 9 files changed, 235 insertions(+), 21 deletions(-)

diff --git a/config/common_base b/config/common_base
index 36ab13d..7c015f9 100644
--- a/config/common_base
+++ b/config/common_base
@@ -347,7 +347,7 @@ CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
 #
 # Compile burst-oriented IPN3KE PMD driver
 #
-CONFIG_RTE_LIBRTE_IPN3KE_PMD=y
+CONFIG_RTE_LIBRTE_IPN3KE_PMD=n
 
 #
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
diff --git a/config/common_linux b/config/common_linux
index a78b8c6..c5cf3d6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -73,4 +73,4 @@ CONFIG_RTE_LIBRTE_HNS3_PMD=y
 # Compile PMD for Intel FPGA raw device
 # To compile, CONFIG_RTE_EAL_VFIO should be enabled.
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
diff --git a/drivers/meson.build b/drivers/meson.build
index 823e3ef..b7fdfb7 100644
--- a/drivers/meson.build
+++ b/drivers/meson.build
@@ -9,12 +9,12 @@ endif
 dpdk_driver_classes = ['common',
 	       'bus',
 	       'mempool', # depends on common and bus.
-	       'net',     # depends on common, bus and mempool.
+	       'raw',     # depends on common and bus.
+	       'net',     # depends on common, bus, mempool and raw.
 	       'crypto',  # depends on common, bus and mempool (net in future).
 	       'compress', # depends on common, bus, mempool.
 	       'event',   # depends on common, bus, mempool and net.
-	       'baseband', # depends on common and bus.
-	       'raw']     # depends on common, bus, mempool, net and event.
+	       'baseband'] # depends on common and bus.
 
 disabled_drivers = get_option('disable_drivers').split(',')
 
diff --git a/drivers/net/ipn3ke/meson.build b/drivers/net/ipn3ke/meson.build
index 74b4d7c..e3c8a67 100644
--- a/drivers/net/ipn3ke/meson.build
+++ b/drivers/net/ipn3ke/meson.build
@@ -8,10 +8,22 @@
 #  rte_eth_dev_destroy()
 #  rte_eth_switch_domain_free()
 #
-allow_experimental_apis = true
 
-sources += files('ipn3ke_ethdev.c',
-	'ipn3ke_representor.c',
-	'ipn3ke_tm.c',
-	'ipn3ke_flow.c')
-deps += ['bus_ifpga', 'sched']
+dep = dependency('libfdt', required: false)
+if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
+	build = false
+	reason = 'missing dependency, "libfdt"'
+endif
+
+if build
+	allow_experimental_apis = true
+
+	sources += files('ipn3ke_ethdev.c',
+		'ipn3ke_representor.c',
+		'ipn3ke_tm.c',
+		'ipn3ke_flow.c')
+	deps += ['bus_ifpga', 'sched', 'pmd_i40e', 'rawdev', 'rawdev_ifpga']
+endif
diff --git a/drivers/raw/ifpga/base/meson.build b/drivers/raw/ifpga/base/meson.build
index 69f6598..b13e13e 100644
--- a/drivers/raw/ifpga/base/meson.build
+++ b/drivers/raw/ifpga/base/meson.build
@@ -25,5 +25,5 @@ sources = [
 
 base_lib = static_library('ifpga_rawdev_base', sources,
 	dependencies: static_rte_eal,
-	c_args: c_args)
+	c_args: cflags)
 base_objs = base_lib.extract_all_objects()
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
index 0ab6fd7..69debff 100644
--- a/drivers/raw/ifpga/meson.build
+++ b/drivers/raw/ifpga/meson.build
@@ -3,18 +3,27 @@
 
 version = 1
 
-subdir('base')
-objs = [base_objs]
-
 dep = dependency('libfdt', required: false)
 if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
 	build = false
 	reason = 'missing dependency, "libfdt"'
 endif
-deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
-	'bus_vdev', 'bus_ifpga', 'net']
-sources = files('ifpga_rawdev.c')
 
-includes += include_directories('base')
+if build
+	subdir('base')
+	objs = [base_objs]
+
+	deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
+		'bus_vdev', 'bus_ifpga', 'net']
+	ext_deps += dep
 
-allow_experimental_apis = true
+	sources = files('ifpga_rawdev.c')
+
+	includes += include_directories('base')
+	includes += include_directories('../../net/ipn3ke')
+
+	allow_experimental_apis = true
+endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 683e3a4..a278552 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -330,7 +330,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 06/19] raw/ifpga/base: align the send buffer for SPI

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 07/19] raw/ifpga/base: add sensor support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 08/19] raw/ifpga/base: introducing sensor APIs

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 09/19] raw/ifpga/base: update SEU register definition

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 10/19] raw/ifpga: add SEU error handler

[Rosen Xu]

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 241 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 241 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 1825143..977dfcf 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -27,6 +27,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -605,6 +607,232 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr)
+		fme_err_read_seu_emr(mgr);
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err)
+		fme_err_read_seu_emr(mgr);
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -652,6 +880,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -677,6 +906,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -696,6 +929,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -720,6 +954,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v18 10/19] raw/ifpga: add SEU error handler

[Thomas Monjalon]

Rosen (and most of your colleagues),
Please use --in-reply-to when sending a new version.
All versions must be a reply to the very first cover letter.

14/11/2019 10:02, Rosen Xu:
> +	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);

Using %lx is usually wrong.
val is 64-bit.
On 32-bit machines (like some Intel ones), it does not compile.
I had to fix it while pulling next-net.

The same issue was in "raw/ifpga/base: clean fme errors".

Please check 32-bit compilation.

Note: we should have such check in the CI.

Re: [dpdk-dev] [PATCH v18 10/19] raw/ifpga: add SEU error handler

[David Marchand]

On Wed, Nov 20, 2019 at 10:23 PM Thomas Monjalon <thomas@monjalon.net> wrote:
>
> Rosen (and most of your colleagues),
> Please use --in-reply-to when sending a new version.
> All versions must be a reply to the very first cover letter.
>
> 14/11/2019 10:02, Rosen Xu:
> > +     IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
>
> Using %lx is usually wrong.
> val is 64-bit.
> On 32-bit machines (like some Intel ones), it does not compile.
> I had to fix it while pulling next-net.
>
> The same issue was in "raw/ifpga/base: clean fme errors".
>
> Please check 32-bit compilation.
>
> Note: we should have such check in the CI.

https://git.dpdk.org/dpdk/commit/?id=3be76aa9294f3788b4f9c615642e6027f1b7948a

-- 
David Marchand

Re: [dpdk-dev] [PATCH v18 10/19] raw/ifpga: add SEU error handler

[Zhang, Tianfei]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Thursday, November 21, 2019 5:30 AM
> To: Thomas Monjalon <thomas@monjalon.net>
> Cc: Xu, Rosen <rosen.xu@intel.com>; dev <dev@dpdk.org>; Zhang, Tianfei
> <tianfei.zhang@intel.com>; Pei, Andy <andy.pei@intel.com>; Ye, Xiaolong
> <xiaolong.ye@intel.com>; Yigit, Ferruh <ferruh.yigit@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v18 10/19] raw/ifpga: add SEU error handler
> 
> On Wed, Nov 20, 2019 at 10:23 PM Thomas Monjalon <thomas@monjalon.net>
> wrote:
> >
> > Rosen (and most of your colleagues),
> > Please use --in-reply-to when sending a new version.
> > All versions must be a reply to the very first cover letter.
> >
> > 14/11/2019 10:02, Rosen Xu:
> > > +     IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
> >
> > Using %lx is usually wrong.
> > val is 64-bit.
> > On 32-bit machines (like some Intel ones), it does not compile.
> > I had to fix it while pulling next-net.
> >
> > The same issue was in "raw/ifpga/base: clean fme errors".
> >
> > Please check 32-bit compilation.

You are correct, Thanks your point out and help to fix it.

> >
> > Note: we should have such check in the CI.
> 
> https://git.dpdk.org/dpdk/commit/?id=3be76aa9294f3788b4f9c615642e6027f1b
> 7948a
> 
> --
> David Marchand

Re: [dpdk-dev] [PATCH v18 10/19] raw/ifpga: add SEU error handler

[Ye Xiaolong]

On 11/20, Thomas Monjalon wrote:
>Rosen (and most of your colleagues),
>Please use --in-reply-to when sending a new version.
>All versions must be a reply to the very first cover letter.

I'll broadcast this message to our team.

Thanks,
Xiaolong

>
>14/11/2019 10:02, Rosen Xu:
>> +	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
>
>Using %lx is usually wrong.
>val is 64-bit.
>On 32-bit machines (like some Intel ones), it does not compile.
>I had to fix it while pulling next-net.
>
>The same issue was in "raw/ifpga/base: clean fme errors".
>
>Please check 32-bit compilation.
>
>Note: we should have such check in the CI.
>
>

Re: [dpdk-dev] [PATCH v18 10/19] raw/ifpga: add SEU error handler

[Xu, Rosen]

Hi,

> -----Original Message-----
> From: Thomas Monjalon [mailto:thomas@monjalon.net]
> Sent: Thursday, November 21, 2019 5:23
> To: Xu, Rosen <rosen.xu@intel.com>
> Cc: dev@dpdk.org; Zhang, Tianfei <tianfei.zhang@intel.com>; Pei, Andy
> <andy.pei@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>; Yigit, Ferruh
> <ferruh.yigit@intel.com>; david.marchand@redhat.com
> Subject: Re: [dpdk-dev] [PATCH v18 10/19] raw/ifpga: add SEU error handler
> 
> Rosen (and most of your colleagues),
> Please use --in-reply-to when sending a new version.
> All versions must be a reply to the very first cover letter.

Got, thanks your reminder.

> 14/11/2019 10:02, Rosen Xu:
> > +	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
> 
> Using %lx is usually wrong.
> val is 64-bit.
> On 32-bit machines (like some Intel ones), it does not compile.
> I had to fix it while pulling next-net.
> 
> The same issue was in "raw/ifpga/base: clean fme errors".
> 
> Please check 32-bit compilation.
> 
> Note: we should have such check in the CI.
> 

[dpdk-dev] [PATCH v18 11/19] raw/ifpga: add PCIe BDF devices tree scan

[Rosen Xu]

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 551 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 562 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 977dfcf..8d633e0 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_malloc.h>
@@ -17,7 +19,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -25,6 +27,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -37,6 +40,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -64,6 +73,494 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(
+		struct ifpga_rawdev *ifpga_rdev, int force_disable);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+		int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
+					sensor->name, value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(
+	struct ifpga_rawdev *ifpga_rdev, int force_disable)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable || force_disable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev,
+					gsd_enable);
+				if (ret == 1 && !gsd_enable) {
+					gsd_enable = 1;
+					i = -1;
+				}
+			}
+		}
+
+		if (gsd_enable)
+			printf(">>>>>>Pls Shutdown APP\n");
+
+		rte_delay_us(100 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -372,8 +869,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -838,6 +1336,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -851,7 +1350,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -865,6 +1364,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -983,6 +1490,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1014,13 +1522,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1034,7 +1561,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1061,6 +1589,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 12/19] net/ipn3ke: remove configuration for i40e port bonding

[Rosen Xu]

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/ipn3ke/Makefile                   |   1 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c            | 292 ++++----------------------
 drivers/net/ipn3ke/ipn3ke_flow.c              |   6 +
 drivers/net/ipn3ke/ipn3ke_rawdev_api.h        |  12 ++
 drivers/net/ipn3ke/ipn3ke_representor.c       |  10 +-
 drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map |   6 +
 drivers/raw/ifpga/Makefile                    |   5 +
 drivers/raw/ifpga/ifpga_rawdev.c              |   4 +
 drivers/raw/ifpga/meson.build                 |   5 +-
 9 files changed, 85 insertions(+), 256 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..8c58d71 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,7 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index af87fda..5b5510f 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -35,6 +36,8 @@
 	{ 0, 0 /* sentinel */ },
 };
 
+struct ipn3ke_pub_func ipn3ke_bridge_func;
+
 static int
 ipn3ke_indirect_read(struct ipn3ke_hw *hw, uint32_t *rd_data,
 	uint32_t addr, uint32_t dev_sel, uint32_t eth_group_sel)
@@ -324,7 +327,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -342,6 +346,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -409,9 +419,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -462,7 +469,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -489,7 +500,14 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	if (ipn3ke_bridge_func.get_ifpga_rawdev == NULL)
+		return -ENOMEM;
+	ifpga_dev = ipn3ke_bridge_func.get_ifpga_rawdev(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -501,6 +519,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -508,6 +542,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -553,253 +588,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_flow.c b/drivers/net/ipn3ke/ipn3ke_flow.c
index 9fc3c8b..f857e64 100644
--- a/drivers/net/ipn3ke/ipn3ke_flow.c
+++ b/drivers/net/ipn3ke/ipn3ke_flow.c
@@ -18,6 +18,12 @@
 #include <rte_malloc.h>
 #include <rte_eth_ctrl.h>
 #include <rte_tailq.h>
+#include <rte_rawdev.h>
+#include <rte_rawdev_pmd.h>
+#include <rte_bus_ifpga.h>
+#include <ifpga_common.h>
+#include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
diff --git a/drivers/net/ipn3ke/ipn3ke_rawdev_api.h b/drivers/net/ipn3ke/ipn3ke_rawdev_api.h
index 671fae8..fd2393f 100644
--- a/drivers/net/ipn3ke/ipn3ke_rawdev_api.h
+++ b/drivers/net/ipn3ke/ipn3ke_rawdev_api.h
@@ -59,4 +59,16 @@ struct ifpga_rawdevg_link_info {
 	enum ifpga_rawdev_link_speed link_speed;
 };
 
+struct ipn3ke_pub_func {
+	struct ifpga_rawdev *(*get_ifpga_rawdev)(const struct rte_rawdev *rdv);
+	int (*set_i40e_sw_dev)(uint16_t port_id, struct rte_eth_dev *sw_dev);
+};
+
+/**
+ * @internal
+ * The publid functions of bridge PAC N3000 FPGA and I40e.
+ */
+extern struct ipn3ke_pub_func ipn3ke_bridge_func;
+
+
 #endif /* _IFPGA_RAWDEV_H_ */
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index d37f5e2..8d9ebef 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -2914,12 +2914,18 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	if (representor_param->port_id >= representor_param->hw->port_num)
 		return -ENODEV;
 
+	if (ipn3ke_bridge_func.set_i40e_sw_dev == NULL)
+		return -ENOMEM;
+
 	rpst->ethdev = ethdev;
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		ipn3ke_bridge_func.set_i40e_sw_dev(rpst->i40e_pf_eth_port_id,
+					    rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
diff --git a/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map b/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
index fc8c95e..fbb74ee 100644
--- a/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
+++ b/drivers/net/ipn3ke/rte_pmd_ipn3ke_version.map
@@ -2,3 +2,9 @@ DPDK_19.05 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	ipn3ke_bridge_func;
+} DPDK_19.05;
diff --git a/drivers/raw/ifpga/Makefile b/drivers/raw/ifpga/Makefile
index 655b292..8fcdb09 100644
--- a/drivers/raw/ifpga/Makefile
+++ b/drivers/raw/ifpga/Makefile
@@ -13,6 +13,7 @@ CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
 CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga_rawdev
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 CFLAGS += -I$(RTE_SDK)/drivers/net/ipn3ke
 LDLIBS += -lrte_eal
 LDLIBS += -lrte_rawdev
@@ -20,6 +21,10 @@ LDLIBS += -lrte_bus_vdev
 LDLIBS += -lrte_kvargs
 LDLIBS += -lrte_bus_pci
 LDLIBS += -lrte_bus_ifpga
+LDLIBS += -lpthread
+LDLIBS += -lfdt
+LDLIBS += -lrte_pmd_i40e
+LDLIBS += -lrte_pmd_ipn3ke
 
 EXPORT_MAP := rte_rawdev_ifpga_version.map
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 8d633e0..c6c69c2 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -28,6 +28,7 @@
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
 #include <rte_string_fns.h>
+#include <rte_pmd_i40e.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -1364,6 +1365,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	ipn3ke_bridge_func.get_ifpga_rawdev = ifpga_rawdev_get;
+	ipn3ke_bridge_func.set_i40e_sw_dev = rte_pmd_i40e_set_switch_dev;
+
 	dev = ifpga_rawdev_allocate(rawdev);
 	if (dev == NULL) {
 		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
index 69debff..c3459e2 100644
--- a/drivers/raw/ifpga/meson.build
+++ b/drivers/raw/ifpga/meson.build
@@ -16,14 +16,15 @@ if build
 	subdir('base')
 	objs = [base_objs]
 
-	deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
-		'bus_vdev', 'bus_ifpga', 'net']
+	deps += ['ethdev', 'rawdev', 'pci', 'bus_pci', 'kvargs',
+		'bus_vdev', 'bus_ifpga', 'net', 'i40e', 'ipn3ke']
 	ext_deps += dep
 
 	sources = files('ifpga_rawdev.c')
 
 	includes += include_directories('base')
 	includes += include_directories('../../net/ipn3ke')
+	includes += include_directories('../../net/i40e')
 
 	allow_experimental_apis = true
 endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 13/19] raw/ifpga/base: add secure support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 137 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 ++++++++++++-----
 4 files changed, 198 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..748ab56 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,63 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager")) {
+			sysmgr_offset = offset;
+			break;
+		}
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +576,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v18 13/19] raw/ifpga/base: add secure support

[Zhang, Tianfei]

> -----Original Message-----
> From: Xu, Rosen
> Sent: Thursday, November 14, 2019 5:03 PM
> To: dev@dpdk.org
> Cc: Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>;
> Pei, Andy <andy.pei@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>; Yigit,
> Ferruh <ferruh.yigit@intel.com>
> Subject: [PATCH v18 13/19] raw/ifpga/base: add secure support
> 
> From: Tianfei zhang <tianfei.zhang@intel.com>
> 
> Add secure max10 device support.

In PAC N3000 Card, it implements the secure functionality on the MAX10 Board Management Controller (BMC) as Root of Trust (RoT). It changes to MAX10 (RTL and Nios FW) to enable secure RSU (Remote System Update) authentication and integrity checks for FPGA Flat image, and FW updates to the card. The card's BMC continues to support features such as power sequence management, board monitoring via sensors, JTAG management and in-band SPI interface access. The external Flash on the card shall be programmed with the Intel root public key hash (for BMC images and FW) and Customer/User key (for FPGA Flat image) during manufacturing, so the image updates (RSU) shall be verified using ECDSA-256 P-256 and SHA2-256 before being written to the MAX10 Image Flash or FPGA Image Flash.

This patch add RoT support for MAX10 because some registers and the content of Device Tree have changed between RoT solution and Non-RoT solution.


> 
> Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
>  drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
>  drivers/raw/ifpga/base/opae_intel_max10.c | 137
> +++++++++++++++++++++++++-----
> drivers/raw/ifpga/base/opae_intel_max10.h |  80 ++++++++++++-----
>  4 files changed, 198 insertions(+), 47 deletions(-)
> 
> diff --git a/drivers/raw/ifpga/base/ifpga_defines.h
> b/drivers/raw/ifpga/base/ifpga_defines.h
> index 8993cc6..1e84b15 100644
> --- a/drivers/raw/ifpga/base/ifpga_defines.h
> +++ b/drivers/raw/ifpga/base/ifpga_defines.h
> @@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
>  	u32 patch_version;
>  	u32 minor_version;
>  	u32 major_version;
> +	u32 max10_version;
> +	u32 nios_fw_version;
>  	u32 nums_of_retimer;
>  	u32 ports_per_retimer;
>  	u32 nums_of_fvl;
> diff --git a/drivers/raw/ifpga/base/ifpga_fme.c
> b/drivers/raw/ifpga/base/ifpga_fme.c
> index 794ca09..87fa596 100644
> --- a/drivers/raw/ifpga/base/ifpga_fme.c
> +++ b/drivers/raw/ifpga/base/ifpga_fme.c
> @@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,  static int
> fme_get_board_interface(struct ifpga_fme_hw *fme)  {
>  	struct fme_bitstream_id id;
> +	u32 val;
> 
>  	if (fme_hdr_get_bitstream_id(fme, &id.id))
>  		return -EINVAL;
> @@ -850,6 +851,18 @@ static int fme_get_board_interface(struct
> ifpga_fme_hw *fme)
>  			fme->board_info.nums_of_fvl,
>  			fme->board_info.ports_per_fvl);
> 
> +	if (max10_sys_read(MAX10_BUILD_VER, &val))
> +		return -EINVAL;
> +	fme->board_info.max10_version = val & 0xffffff;
> +
> +	if (max10_sys_read(NIOS2_FW_VERSION, &val))
> +		return -EINVAL;
> +	fme->board_info.nios_fw_version = val & 0xffffff;
> +
> +	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
> +		fme->board_info.max10_version,
> +		fme->board_info.nios_fw_version);
> +
>  	return 0;
>  }
> 
> @@ -858,16 +871,11 @@ static int spi_self_checking(void)
>  	u32 val;
>  	int ret;
> 
> -	ret = max10_reg_read(0x30043c, &val);
> +	ret = max10_sys_read(MAX10_TEST_REG, &val);
>  	if (ret)
>  		return -EIO;
> 
> -	if (val != 0x87654321) {
> -		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
> -		return -EIO;
> -	}
> -
> -	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
> +	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
> 
>  	return 0;
>  }
> @@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct
> ifpga_fme_hw *fme,
>  	if (!dev)
>  		return -ENODEV;
> 
> -	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
> +	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
>  		dev_err(dev, "%s: read pkvl status fail\n", __func__);
>  		return -EINVAL;
>  	}
> @@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw
> *fme,
>  	if (!dev)
>  		return -ENODEV;
> 
> -	if (max10_reg_read(sensor->value_reg, value)) {
> +	if (max10_sys_read(sensor->value_reg, value)) {
>  		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
>  				__func__, sensor->value_reg);
>  		return -EINVAL;
> diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c
> b/drivers/raw/ifpga/base/opae_intel_max10.c
> index ae7a8df..748ab56 100644
> --- a/drivers/raw/ifpga/base/opae_intel_max10.c
> +++ b/drivers/raw/ifpga/base/opae_intel_max10.c
> @@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
>  			reg, 4, (unsigned char *)&tmp);
>  }
> 
> +int max10_sys_read(unsigned int offset, unsigned int *val) {
> +	if (!g_max10)
> +		return -ENODEV;
> +
> +	return max10_reg_read(g_max10->base + offset, val); }
> +
> +int max10_sys_write(unsigned int offset, unsigned int val) {
> +	if (!g_max10)
> +		return -ENODEV;
> +
> +	return max10_reg_write(g_max10->base + offset, val); }
> +
>  static struct max10_compatible_id max10_id_table[] = {
>  	{.compatible = MAX10_PAC,},
>  	{.compatible = MAX10_PAC_N3000,},
> @@ -66,7 +82,8 @@ static void max10_check_capability(struct
> intel_max10_device *max10)
>  		max10->flags |= MAX10_FLAGS_NO_I2C2 |
>  				MAX10_FLAGS_NO_BMCIMG_FLASH;
>  		dev_info(max10, "found %s card\n", max10->id->compatible);
> -	}
> +	} else
> +		max10->flags |= MAX10_FLAGS_MAC_CACHE;
>  }
> 
>  static int altera_nor_flash_read(u32 offset, @@ -100,7 +117,7 @@ static int
> enable_nor_flash(bool on)
>  	unsigned int val = 0;
>  	int ret;
> 
> -	ret = max10_reg_read(RSU_REG_OFF, &val);
> +	ret = max10_sys_read(RSU_REG, &val);
>  	if (ret) {
>  		dev_err(NULL "enabling flash error\n");
>  		return ret;
> @@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
>  	else
>  		val &= ~RSU_ENABLE;
> 
> -	return max10_reg_write(RSU_REG_OFF, val);
> +	return max10_sys_write(RSU_REG, val);
>  }
> 
>  static int init_max10_device_table(struct intel_max10_device *max10) @@
> -123,7 +140,7 @@ static int init_max10_device_table(struct
> intel_max10_device *max10)
>  	u32 dt_size, dt_addr, val;
>  	int ret;
> 
> -	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
> +	ret = max10_sys_read(DT_AVAIL_REG, &val);
>  	if (ret) {
>  		dev_err(max10 "cannot read DT_AVAIL_REG\n");
>  		return ret;
> @@ -134,7 +151,7 @@ static int init_max10_device_table(struct
> intel_max10_device *max10)
>  		return -EINVAL;
>  	}
> 
> -	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
> +	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
>  	if (ret) {
>  		dev_info(max10 "cannot get base addr of device table\n");
>  		return ret;
> @@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info
> *info,
>  		if (!sensor_reg_valid(&info->regs[i]))
>  			continue;
> 
> -		ret = max10_reg_read(info->regs[i].regoff, &val);
> +		ret = max10_sys_read(info->regs[i].regoff, &val);
>  		if (ret)
>  			break;
> 
> @@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info
> *info,
>  	return ret;
>  }
> 
> -static int max10_sensor_init(struct intel_max10_device *dev)
> +static int
> +max10_sensor_init(struct intel_max10_device *dev, int parent)
>  {
>  	int i, ret = 0, offset = 0;
>  	const fdt32_t *num;
> @@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device
> *dev)
>  		return 0;
>  	}
> 
> -	fdt_for_each_subnode(offset, fdt_root, 0) {
> +	fdt_for_each_subnode(offset, fdt_root, parent) {
>  		ptr = fdt_get_name(fdt_root, offset, NULL);
>  		if (!ptr) {
>  			dev_err(dev, "failed to fdt get name\n"); @@ -417,7 +435,16
> @@ static int max10_sensor_init(struct intel_max10_device *dev)
>  				continue;
>  			}
> 
> -			raw->regs[i].regoff = start;
> +			/* This is a hack to compatible with non-secure
> +			 * solution. If sensors are included in root node,
> +			 * then it's non-secure dtb, which use absolute addr
> +			 * of non-secure solution.
> +			 */
> +			if (parent)
> +				raw->regs[i].regoff = start;
> +			else
> +				raw->regs[i].regoff = start -
> +					MAX10_BASE_ADDR;
>  			raw->regs[i].size = size;
>  		}
> 
> @@ -469,6 +496,63 @@ static int max10_sensor_init(struct
> intel_max10_device *dev)
>  	return ret;
>  }
> 
> +static int check_max10_version(struct intel_max10_device *dev) {
> +	unsigned int v;
> +
> +	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
> +				&v)) {
> +		if (v != 0xffffffff) {
> +			dev_info(dev, "secure MAX10 detected\n");
> +			dev->base = MAX10_SEC_BASE_ADDR;
> +			dev->flags |= MAX10_FLAGS_SECURE;
> +		} else {
> +			dev_info(dev, "non-secure MAX10 detected\n");
> +			dev->base = MAX10_BASE_ADDR;
> +		}
> +		return 0;
> +	}
> +
> +	return -ENODEV;
> +}
> +
> +static int
> +max10_secure_hw_init(struct intel_max10_device *dev) {
> +	int offset, sysmgr_offset = 0;
> +	char *fdt_root;
> +
> +	fdt_root = dev->fdt_root;
> +	if (!fdt_root) {
> +		dev_debug(dev, "skip init as not find Device Tree\n");
> +		return 0;
> +	}
> +
> +	fdt_for_each_subnode(offset, fdt_root, 0) {
> +		if (!fdt_node_check_compatible(fdt_root, offset,
> +					"intel-max10,system-manager")) {
> +			sysmgr_offset = offset;
> +			break;
> +		}
> +	}
> +
> +	max10_check_capability(dev);
> +
> +	max10_sensor_init(dev, sysmgr_offset);
> +
> +	return 0;
> +}
> +
> +static int
> +max10_non_secure_hw_init(struct intel_max10_device *dev) {
> +	max10_check_capability(dev);
> +
> +	max10_sensor_init(dev, 0);
> +
> +	return 0;
> +}
> +
>  struct intel_max10_device *
>  intel_max10_device_probe(struct altera_spi_device *spi,
>  		int chipselect)
> @@ -492,32 +576,47 @@ struct intel_max10_device *
>  	/* set the max10 device firstly */
>  	g_max10 = dev;
> 
> -	/* init the MAX10 device table */
> +	/* check the max10 version */
> +	ret = check_max10_version(dev);
> +	if (ret) {
> +		dev_err(dev, "Failed to find max10 hardware!\n");
> +		goto free_dev;
> +	}
> +
> +	/* load the MAX10 device table */
>  	ret = init_max10_device_table(dev);
>  	if (ret) {
> -		dev_err(dev, "init max10 device table fail\n");
> +		dev_err(dev, "Init max10 device table fail\n");
>  		goto free_dev;
>  	}
> 
> -	max10_check_capability(dev);
> +	/* init max10 devices, like sensor*/
> +	if (dev->flags & MAX10_FLAGS_SECURE)
> +		ret = max10_secure_hw_init(dev);
> +	else
> +		ret = max10_non_secure_hw_init(dev);
> +	if (ret) {
> +		dev_err(dev, "Failed to init max10 hardware!\n");
> +		goto free_dtb;
> +	}
> 
>  	/* read FPGA loading information */
> -	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
> +	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
>  	if (ret) {
>  		dev_err(dev, "fail to get FPGA loading info\n");
> -		goto spi_tran_fail;
> +		goto release_max10_hw;
>  	}
>  	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
> 
> -
> -	max10_sensor_init(dev);
> -
>  	return dev;
> 
> -spi_tran_fail:
> +release_max10_hw:
> +	max10_sensor_uinit();
> +free_dtb:
>  	if (dev->fdt_root)
>  		opae_free(dev->fdt_root);
> -	spi_transaction_remove(dev->spi_tran_dev);
> +	if (dev->spi_tran_dev)
> +		spi_transaction_remove(dev->spi_tran_dev);
>  free_dev:
>  	g_max10 = NULL;
>  	opae_free(dev);
> diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h
> b/drivers/raw/ifpga/base/opae_intel_max10.h
> index 90bf098..e632941 100644
> --- a/drivers/raw/ifpga/base/opae_intel_max10.h
> +++ b/drivers/raw/ifpga/base/opae_intel_max10.h
> @@ -23,6 +23,8 @@ struct max10_compatible_id {
>  #define MAX10_FLAGS_SPI                 BIT(3)
>  #define MAX10_FLGAS_NIOS_SPI            BIT(4)
>  #define MAX10_FLAGS_PKVL                BIT(5)
> +#define MAX10_FLAGS_SECURE		BIT(6)
> +#define MAX10_FLAGS_MAC_CACHE		BIT(7)
> 
>  struct intel_max10_device {
>  	unsigned int flags; /*max10 hardware capability*/ @@ -30,6 +32,7 @@
> struct intel_max10_device {
>  	struct spi_transaction_dev *spi_tran_dev;
>  	struct max10_compatible_id *id; /*max10 compatible*/
>  	char *fdt_root;
> +	unsigned int base; /* max10 base address */
>  };
> 
>  /* retimer speed */
> @@ -74,30 +77,69 @@ struct opae_retimer_status {  #define FLASH_BASE
> 0x10000000  #define FLASH_OPTION_BITS 0x10000
> 
> -#define NIOS2_FW_VERSION_OFF   0x300400
> -#define RSU_REG_OFF            0x30042c
> -#define FPGA_RP_LOAD		BIT(3)
> -#define NIOS2_PRERESET		BIT(4)
> -#define NIOS2_HANG		BIT(5)
> -#define RSU_ENABLE		BIT(6)
> -#define NIOS2_RESET		BIT(7)
> -#define NIOS2_I2C2_POLL_STOP	BIT(13)
> -#define FPGA_RECONF_REG_OFF	0x300430
> -#define COUNTDOWN_START		BIT(18)
> -#define MAX10_BUILD_VER_OFF	0x300468
> -#define PCB_INFO		GENMASK(31, 24)
> -#define MAX10_BUILD_VERION	GENMASK(23, 0)
> -#define FPGA_PAGE_INFO_OFF	0x30046c
> -#define DT_AVAIL_REG_OFF	0x300490
> -#define DT_AVAIL		BIT(0)
> -#define DT_BASE_ADDR_REG_OFF	0x300494
> -#define PKVL_POLLING_CTRL       0x300480
> -#define PKVL_LINK_STATUS        0x300564
> +/* System Registers */
> +#define MAX10_BASE_ADDR		0x300400
> +#define MAX10_SEC_BASE_ADDR	0x300800
> +/* Register offset of system registers */
> +#define NIOS2_FW_VERSION	0x0
> +#define MAX10_MACADDR1		0x10
> +#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
> +#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
> +#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
> +#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
> +#define MAX10_MACADDR2		0x14
> +#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
> +#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
> +#define   MAX10_MAC_COUNT	GENMASK(23, 16)
> +#define RSU_REG			0x2c
> +#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
> +#define   FPGA_RP_LOAD		BIT(3)
> +#define   NIOS2_PRERESET	BIT(4)
> +#define   NIOS2_HANG		BIT(5)
> +#define   RSU_ENABLE		BIT(6)
> +#define   NIOS2_RESET		BIT(7)
> +#define   NIOS2_I2C2_POLL_STOP	BIT(13)
> +#define   PKVL_EEPROM_LOAD	BIT(31)
> +#define FPGA_RECONF_REG		0x30
> +#define MAX10_TEST_REG		0x3c
> +#define   COUNTDOWN_START	BIT(18)
> +#define MAX10_BUILD_VER		0x68
> +#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
> +#define   PCB_INFO		GENMASK(31, 24)
> +#define FPGA_PAGE_INFO		0x6c
> +#define DT_AVAIL_REG		0x90
> +#define   DT_AVAIL		BIT(0)
> +#define DT_BASE_ADDR_REG	0x94
> +#define MAX10_DOORBELL		0x400
> +#define   RSU_REQUEST		BIT(0)
> +#define   SEC_PROGRESS		GENMASK(7, 4)
> +#define   HOST_STATUS		GENMASK(11, 8)
> +#define   SEC_STATUS		GENMASK(23, 16)
> +
> +/* PKVL related registers, in system register region */
> +#define PKVL_POLLING_CTRL		0x80
> +#define   POLLING_MODE			GENMASK(15, 0)
> +#define   PKVL_A_PRELOAD		BIT(16)
> +#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
> +#define   PKVL_A_DATA_TOO_BIG		BIT(18)
> +#define   PKVL_A_HDR_CHECKSUM		BIT(20)
> +#define   PKVL_B_PRELOAD		BIT(24)
> +#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
> +#define   PKVL_B_DATA_TOO_BIG		BIT(26)
> +#define   PKVL_B_HDR_CHECKSUM		BIT(28)
> +#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
> +#define PKVL_LINK_STATUS		0x164
> +#define PKVL_A_VERSION			0x254
> +#define PKVL_B_VERSION			0x258
> +#define   SERDES_VERSION		GENMASK(15, 0)
> +#define   SBUS_VERSION			GENMASK(31, 16)
> 
>  #define DFT_MAX_SIZE		0x7e0000
> 
>  int max10_reg_read(unsigned int reg, unsigned int *val);  int
> max10_reg_write(unsigned int reg, unsigned int val);
> +int max10_sys_read(unsigned int offset, unsigned int *val); int
> +max10_sys_write(unsigned int offset, unsigned int val);
>  struct intel_max10_device *
>  intel_max10_device_probe(struct altera_spi_device *spi,
>  		int chipselect);
> --
> 1.8.3.1

Re: [dpdk-dev] [PATCH v18 13/19] raw/ifpga/base: add secure support

[Ferruh Yigit]

On 11/14/2019 11:05 PM, Zhang, Tianfei wrote:
> 
>> -----Original Message-----
>> From: Xu, Rosen
>> Sent: Thursday, November 14, 2019 5:03 PM
>> To: dev@dpdk.org
>> Cc: Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>;
>> Pei, Andy <andy.pei@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>; Yigit,
>> Ferruh <ferruh.yigit@intel.com>
>> Subject: [PATCH v18 13/19] raw/ifpga/base: add secure support
>>
>> From: Tianfei zhang <tianfei.zhang@intel.com>
>>
>> Add secure max10 device support.
> 
> In PAC N3000 Card, it implements the secure functionality on the MAX10 Board Management Controller (BMC) as Root of Trust (RoT). It changes to MAX10 (RTL and Nios FW) to enable secure RSU (Remote System Update) authentication and integrity checks for FPGA Flat image, and FW updates to the card. The card's BMC continues to support features such as power sequence management, board monitoring via sensors, JTAG management and in-band SPI interface access. The external Flash on the card shall be programmed with the Intel root public key hash (for BMC images and FW) and Customer/User key (for FPGA Flat image) during manufacturing, so the image updates (RSU) shall be verified using ECDSA-256 P-256 and SHA2-256 before being written to the MAX10 Image Flash or FPGA Image Flash.
> 
> This patch add RoT support for MAX10 because some registers and the content of Device Tree have changed between RoT solution and Non-RoT solution.

Thanks Tianfei for the additional information, I will update commit as
following, please let me know if you have any objection:

raw/ifpga/base: support max10 security feature

In PAC N3000 Card, MAX10 Board Management Controller (BMC) implements
the security functionality.

Security functionality adds secure Remote System Update (RSU)
authentication and integrity checks for FPGA flat image,
and FW updates to the card.

This patch adds security feature support for MAX10, in secure solution
some registers and the content of the Device Tree changes.

> 
> 
>>
>> Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
>> Signed-off-by: Andy Pei <andy.pei@intel.com>
>> ---
>>  drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
>>  drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
>>  drivers/raw/ifpga/base/opae_intel_max10.c | 137
>> +++++++++++++++++++++++++-----
>> drivers/raw/ifpga/base/opae_intel_max10.h |  80 ++++++++++++-----
>>  4 files changed, 198 insertions(+), 47 deletions(-)
>>
>> diff --git a/drivers/raw/ifpga/base/ifpga_defines.h
>> b/drivers/raw/ifpga/base/ifpga_defines.h
>> index 8993cc6..1e84b15 100644
>> --- a/drivers/raw/ifpga/base/ifpga_defines.h
>> +++ b/drivers/raw/ifpga/base/ifpga_defines.h
>> @@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
>>  	u32 patch_version;
>>  	u32 minor_version;
>>  	u32 major_version;
>> +	u32 max10_version;
>> +	u32 nios_fw_version;
>>  	u32 nums_of_retimer;
>>  	u32 ports_per_retimer;
>>  	u32 nums_of_fvl;
>> diff --git a/drivers/raw/ifpga/base/ifpga_fme.c
>> b/drivers/raw/ifpga/base/ifpga_fme.c
>> index 794ca09..87fa596 100644
>> --- a/drivers/raw/ifpga/base/ifpga_fme.c
>> +++ b/drivers/raw/ifpga/base/ifpga_fme.c
>> @@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,  static int
>> fme_get_board_interface(struct ifpga_fme_hw *fme)  {
>>  	struct fme_bitstream_id id;
>> +	u32 val;
>>
>>  	if (fme_hdr_get_bitstream_id(fme, &id.id))
>>  		return -EINVAL;
>> @@ -850,6 +851,18 @@ static int fme_get_board_interface(struct
>> ifpga_fme_hw *fme)
>>  			fme->board_info.nums_of_fvl,
>>  			fme->board_info.ports_per_fvl);
>>
>> +	if (max10_sys_read(MAX10_BUILD_VER, &val))
>> +		return -EINVAL;
>> +	fme->board_info.max10_version = val & 0xffffff;
>> +
>> +	if (max10_sys_read(NIOS2_FW_VERSION, &val))
>> +		return -EINVAL;
>> +	fme->board_info.nios_fw_version = val & 0xffffff;
>> +
>> +	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
>> +		fme->board_info.max10_version,
>> +		fme->board_info.nios_fw_version);
>> +
>>  	return 0;
>>  }
>>
>> @@ -858,16 +871,11 @@ static int spi_self_checking(void)
>>  	u32 val;
>>  	int ret;
>>
>> -	ret = max10_reg_read(0x30043c, &val);
>> +	ret = max10_sys_read(MAX10_TEST_REG, &val);
>>  	if (ret)
>>  		return -EIO;
>>
>> -	if (val != 0x87654321) {
>> -		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
>> -		return -EIO;
>> -	}
>> -
>> -	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
>> +	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
>>
>>  	return 0;
>>  }
>> @@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct
>> ifpga_fme_hw *fme,
>>  	if (!dev)
>>  		return -ENODEV;
>>
>> -	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
>> +	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
>>  		dev_err(dev, "%s: read pkvl status fail\n", __func__);
>>  		return -EINVAL;
>>  	}
>> @@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw
>> *fme,
>>  	if (!dev)
>>  		return -ENODEV;
>>
>> -	if (max10_reg_read(sensor->value_reg, value)) {
>> +	if (max10_sys_read(sensor->value_reg, value)) {
>>  		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
>>  				__func__, sensor->value_reg);
>>  		return -EINVAL;
>> diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c
>> b/drivers/raw/ifpga/base/opae_intel_max10.c
>> index ae7a8df..748ab56 100644
>> --- a/drivers/raw/ifpga/base/opae_intel_max10.c
>> +++ b/drivers/raw/ifpga/base/opae_intel_max10.c
>> @@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
>>  			reg, 4, (unsigned char *)&tmp);
>>  }
>>
>> +int max10_sys_read(unsigned int offset, unsigned int *val) {
>> +	if (!g_max10)
>> +		return -ENODEV;
>> +
>> +	return max10_reg_read(g_max10->base + offset, val); }
>> +
>> +int max10_sys_write(unsigned int offset, unsigned int val) {
>> +	if (!g_max10)
>> +		return -ENODEV;
>> +
>> +	return max10_reg_write(g_max10->base + offset, val); }
>> +
>>  static struct max10_compatible_id max10_id_table[] = {
>>  	{.compatible = MAX10_PAC,},
>>  	{.compatible = MAX10_PAC_N3000,},
>> @@ -66,7 +82,8 @@ static void max10_check_capability(struct
>> intel_max10_device *max10)
>>  		max10->flags |= MAX10_FLAGS_NO_I2C2 |
>>  				MAX10_FLAGS_NO_BMCIMG_FLASH;
>>  		dev_info(max10, "found %s card\n", max10->id->compatible);
>> -	}
>> +	} else
>> +		max10->flags |= MAX10_FLAGS_MAC_CACHE;
>>  }
>>
>>  static int altera_nor_flash_read(u32 offset, @@ -100,7 +117,7 @@ static int
>> enable_nor_flash(bool on)
>>  	unsigned int val = 0;
>>  	int ret;
>>
>> -	ret = max10_reg_read(RSU_REG_OFF, &val);
>> +	ret = max10_sys_read(RSU_REG, &val);
>>  	if (ret) {
>>  		dev_err(NULL "enabling flash error\n");
>>  		return ret;
>> @@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
>>  	else
>>  		val &= ~RSU_ENABLE;
>>
>> -	return max10_reg_write(RSU_REG_OFF, val);
>> +	return max10_sys_write(RSU_REG, val);
>>  }
>>
>>  static int init_max10_device_table(struct intel_max10_device *max10) @@
>> -123,7 +140,7 @@ static int init_max10_device_table(struct
>> intel_max10_device *max10)
>>  	u32 dt_size, dt_addr, val;
>>  	int ret;
>>
>> -	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
>> +	ret = max10_sys_read(DT_AVAIL_REG, &val);
>>  	if (ret) {
>>  		dev_err(max10 "cannot read DT_AVAIL_REG\n");
>>  		return ret;
>> @@ -134,7 +151,7 @@ static int init_max10_device_table(struct
>> intel_max10_device *max10)
>>  		return -EINVAL;
>>  	}
>>
>> -	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
>> +	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
>>  	if (ret) {
>>  		dev_info(max10 "cannot get base addr of device table\n");
>>  		return ret;
>> @@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info
>> *info,
>>  		if (!sensor_reg_valid(&info->regs[i]))
>>  			continue;
>>
>> -		ret = max10_reg_read(info->regs[i].regoff, &val);
>> +		ret = max10_sys_read(info->regs[i].regoff, &val);
>>  		if (ret)
>>  			break;
>>
>> @@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info
>> *info,
>>  	return ret;
>>  }
>>
>> -static int max10_sensor_init(struct intel_max10_device *dev)
>> +static int
>> +max10_sensor_init(struct intel_max10_device *dev, int parent)
>>  {
>>  	int i, ret = 0, offset = 0;
>>  	const fdt32_t *num;
>> @@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device
>> *dev)
>>  		return 0;
>>  	}
>>
>> -	fdt_for_each_subnode(offset, fdt_root, 0) {
>> +	fdt_for_each_subnode(offset, fdt_root, parent) {
>>  		ptr = fdt_get_name(fdt_root, offset, NULL);
>>  		if (!ptr) {
>>  			dev_err(dev, "failed to fdt get name\n"); @@ -417,7 +435,16
>> @@ static int max10_sensor_init(struct intel_max10_device *dev)
>>  				continue;
>>  			}
>>
>> -			raw->regs[i].regoff = start;
>> +			/* This is a hack to compatible with non-secure
>> +			 * solution. If sensors are included in root node,
>> +			 * then it's non-secure dtb, which use absolute addr
>> +			 * of non-secure solution.
>> +			 */
>> +			if (parent)
>> +				raw->regs[i].regoff = start;
>> +			else
>> +				raw->regs[i].regoff = start -
>> +					MAX10_BASE_ADDR;
>>  			raw->regs[i].size = size;
>>  		}
>>
>> @@ -469,6 +496,63 @@ static int max10_sensor_init(struct
>> intel_max10_device *dev)
>>  	return ret;
>>  }
>>
>> +static int check_max10_version(struct intel_max10_device *dev) {
>> +	unsigned int v;
>> +
>> +	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
>> +				&v)) {
>> +		if (v != 0xffffffff) {
>> +			dev_info(dev, "secure MAX10 detected\n");
>> +			dev->base = MAX10_SEC_BASE_ADDR;
>> +			dev->flags |= MAX10_FLAGS_SECURE;
>> +		} else {
>> +			dev_info(dev, "non-secure MAX10 detected\n");
>> +			dev->base = MAX10_BASE_ADDR;
>> +		}
>> +		return 0;
>> +	}
>> +
>> +	return -ENODEV;
>> +}
>> +
>> +static int
>> +max10_secure_hw_init(struct intel_max10_device *dev) {
>> +	int offset, sysmgr_offset = 0;
>> +	char *fdt_root;
>> +
>> +	fdt_root = dev->fdt_root;
>> +	if (!fdt_root) {
>> +		dev_debug(dev, "skip init as not find Device Tree\n");
>> +		return 0;
>> +	}
>> +
>> +	fdt_for_each_subnode(offset, fdt_root, 0) {
>> +		if (!fdt_node_check_compatible(fdt_root, offset,
>> +					"intel-max10,system-manager")) {
>> +			sysmgr_offset = offset;
>> +			break;
>> +		}
>> +	}
>> +
>> +	max10_check_capability(dev);
>> +
>> +	max10_sensor_init(dev, sysmgr_offset);
>> +
>> +	return 0;
>> +}
>> +
>> +static int
>> +max10_non_secure_hw_init(struct intel_max10_device *dev) {
>> +	max10_check_capability(dev);
>> +
>> +	max10_sensor_init(dev, 0);
>> +
>> +	return 0;
>> +}
>> +
>>  struct intel_max10_device *
>>  intel_max10_device_probe(struct altera_spi_device *spi,
>>  		int chipselect)
>> @@ -492,32 +576,47 @@ struct intel_max10_device *
>>  	/* set the max10 device firstly */
>>  	g_max10 = dev;
>>
>> -	/* init the MAX10 device table */
>> +	/* check the max10 version */
>> +	ret = check_max10_version(dev);
>> +	if (ret) {
>> +		dev_err(dev, "Failed to find max10 hardware!\n");
>> +		goto free_dev;
>> +	}
>> +
>> +	/* load the MAX10 device table */
>>  	ret = init_max10_device_table(dev);
>>  	if (ret) {
>> -		dev_err(dev, "init max10 device table fail\n");
>> +		dev_err(dev, "Init max10 device table fail\n");
>>  		goto free_dev;
>>  	}
>>
>> -	max10_check_capability(dev);
>> +	/* init max10 devices, like sensor*/
>> +	if (dev->flags & MAX10_FLAGS_SECURE)
>> +		ret = max10_secure_hw_init(dev);
>> +	else
>> +		ret = max10_non_secure_hw_init(dev);
>> +	if (ret) {
>> +		dev_err(dev, "Failed to init max10 hardware!\n");
>> +		goto free_dtb;
>> +	}
>>
>>  	/* read FPGA loading information */
>> -	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
>> +	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
>>  	if (ret) {
>>  		dev_err(dev, "fail to get FPGA loading info\n");
>> -		goto spi_tran_fail;
>> +		goto release_max10_hw;
>>  	}
>>  	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
>>
>> -
>> -	max10_sensor_init(dev);
>> -
>>  	return dev;
>>
>> -spi_tran_fail:
>> +release_max10_hw:
>> +	max10_sensor_uinit();
>> +free_dtb:
>>  	if (dev->fdt_root)
>>  		opae_free(dev->fdt_root);
>> -	spi_transaction_remove(dev->spi_tran_dev);
>> +	if (dev->spi_tran_dev)
>> +		spi_transaction_remove(dev->spi_tran_dev);
>>  free_dev:
>>  	g_max10 = NULL;
>>  	opae_free(dev);
>> diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h
>> b/drivers/raw/ifpga/base/opae_intel_max10.h
>> index 90bf098..e632941 100644
>> --- a/drivers/raw/ifpga/base/opae_intel_max10.h
>> +++ b/drivers/raw/ifpga/base/opae_intel_max10.h
>> @@ -23,6 +23,8 @@ struct max10_compatible_id {
>>  #define MAX10_FLAGS_SPI                 BIT(3)
>>  #define MAX10_FLGAS_NIOS_SPI            BIT(4)
>>  #define MAX10_FLAGS_PKVL                BIT(5)
>> +#define MAX10_FLAGS_SECURE		BIT(6)
>> +#define MAX10_FLAGS_MAC_CACHE		BIT(7)
>>
>>  struct intel_max10_device {
>>  	unsigned int flags; /*max10 hardware capability*/ @@ -30,6 +32,7 @@
>> struct intel_max10_device {
>>  	struct spi_transaction_dev *spi_tran_dev;
>>  	struct max10_compatible_id *id; /*max10 compatible*/
>>  	char *fdt_root;
>> +	unsigned int base; /* max10 base address */
>>  };
>>
>>  /* retimer speed */
>> @@ -74,30 +77,69 @@ struct opae_retimer_status {  #define FLASH_BASE
>> 0x10000000  #define FLASH_OPTION_BITS 0x10000
>>
>> -#define NIOS2_FW_VERSION_OFF   0x300400
>> -#define RSU_REG_OFF            0x30042c
>> -#define FPGA_RP_LOAD		BIT(3)
>> -#define NIOS2_PRERESET		BIT(4)
>> -#define NIOS2_HANG		BIT(5)
>> -#define RSU_ENABLE		BIT(6)
>> -#define NIOS2_RESET		BIT(7)
>> -#define NIOS2_I2C2_POLL_STOP	BIT(13)
>> -#define FPGA_RECONF_REG_OFF	0x300430
>> -#define COUNTDOWN_START		BIT(18)
>> -#define MAX10_BUILD_VER_OFF	0x300468
>> -#define PCB_INFO		GENMASK(31, 24)
>> -#define MAX10_BUILD_VERION	GENMASK(23, 0)
>> -#define FPGA_PAGE_INFO_OFF	0x30046c
>> -#define DT_AVAIL_REG_OFF	0x300490
>> -#define DT_AVAIL		BIT(0)
>> -#define DT_BASE_ADDR_REG_OFF	0x300494
>> -#define PKVL_POLLING_CTRL       0x300480
>> -#define PKVL_LINK_STATUS        0x300564
>> +/* System Registers */
>> +#define MAX10_BASE_ADDR		0x300400
>> +#define MAX10_SEC_BASE_ADDR	0x300800
>> +/* Register offset of system registers */
>> +#define NIOS2_FW_VERSION	0x0
>> +#define MAX10_MACADDR1		0x10
>> +#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
>> +#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
>> +#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
>> +#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
>> +#define MAX10_MACADDR2		0x14
>> +#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
>> +#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
>> +#define   MAX10_MAC_COUNT	GENMASK(23, 16)
>> +#define RSU_REG			0x2c
>> +#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
>> +#define   FPGA_RP_LOAD		BIT(3)
>> +#define   NIOS2_PRERESET	BIT(4)
>> +#define   NIOS2_HANG		BIT(5)
>> +#define   RSU_ENABLE		BIT(6)
>> +#define   NIOS2_RESET		BIT(7)
>> +#define   NIOS2_I2C2_POLL_STOP	BIT(13)
>> +#define   PKVL_EEPROM_LOAD	BIT(31)
>> +#define FPGA_RECONF_REG		0x30
>> +#define MAX10_TEST_REG		0x3c
>> +#define   COUNTDOWN_START	BIT(18)
>> +#define MAX10_BUILD_VER		0x68
>> +#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
>> +#define   PCB_INFO		GENMASK(31, 24)
>> +#define FPGA_PAGE_INFO		0x6c
>> +#define DT_AVAIL_REG		0x90
>> +#define   DT_AVAIL		BIT(0)
>> +#define DT_BASE_ADDR_REG	0x94
>> +#define MAX10_DOORBELL		0x400
>> +#define   RSU_REQUEST		BIT(0)
>> +#define   SEC_PROGRESS		GENMASK(7, 4)
>> +#define   HOST_STATUS		GENMASK(11, 8)
>> +#define   SEC_STATUS		GENMASK(23, 16)
>> +
>> +/* PKVL related registers, in system register region */
>> +#define PKVL_POLLING_CTRL		0x80
>> +#define   POLLING_MODE			GENMASK(15, 0)
>> +#define   PKVL_A_PRELOAD		BIT(16)
>> +#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
>> +#define   PKVL_A_DATA_TOO_BIG		BIT(18)
>> +#define   PKVL_A_HDR_CHECKSUM		BIT(20)
>> +#define   PKVL_B_PRELOAD		BIT(24)
>> +#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
>> +#define   PKVL_B_DATA_TOO_BIG		BIT(26)
>> +#define   PKVL_B_HDR_CHECKSUM		BIT(28)
>> +#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
>> +#define PKVL_LINK_STATUS		0x164
>> +#define PKVL_A_VERSION			0x254
>> +#define PKVL_B_VERSION			0x258
>> +#define   SERDES_VERSION		GENMASK(15, 0)
>> +#define   SBUS_VERSION			GENMASK(31, 16)
>>
>>  #define DFT_MAX_SIZE		0x7e0000
>>
>>  int max10_reg_read(unsigned int reg, unsigned int *val);  int
>> max10_reg_write(unsigned int reg, unsigned int val);
>> +int max10_sys_read(unsigned int offset, unsigned int *val); int
>> +max10_sys_write(unsigned int offset, unsigned int val);
>>  struct intel_max10_device *
>>  intel_max10_device_probe(struct altera_spi_device *spi,
>>  		int chipselect);
>> --
>> 1.8.3.1
> 

Re: [dpdk-dev] [PATCH v18 13/19] raw/ifpga/base: add secure support

[Zhang, Tianfei]

> -----Original Message-----
> From: Yigit, Ferruh
> Sent: Friday, November 15, 2019 5:54 PM
> To: Zhang, Tianfei <tianfei.zhang@intel.com>; Xu, Rosen <rosen.xu@intel.com>;
> dev@dpdk.org
> Cc: Pei, Andy <andy.pei@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>
> Subject: Re: [PATCH v18 13/19] raw/ifpga/base: add secure support
> 
> On 11/14/2019 11:05 PM, Zhang, Tianfei wrote:
> >
> >> -----Original Message-----
> >> From: Xu, Rosen
> >> Sent: Thursday, November 14, 2019 5:03 PM
> >> To: dev@dpdk.org
> >> Cc: Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> >> <tianfei.zhang@intel.com>; Pei, Andy <andy.pei@intel.com>; Ye,
> >> Xiaolong <xiaolong.ye@intel.com>; Yigit, Ferruh
> >> <ferruh.yigit@intel.com>
> >> Subject: [PATCH v18 13/19] raw/ifpga/base: add secure support
> >>
> >> From: Tianfei zhang <tianfei.zhang@intel.com>
> >>
> >> Add secure max10 device support.
> >
> > In PAC N3000 Card, it implements the secure functionality on the MAX10
> Board Management Controller (BMC) as Root of Trust (RoT). It changes to
> MAX10 (RTL and Nios FW) to enable secure RSU (Remote System Update)
> authentication and integrity checks for FPGA Flat image, and FW updates to the
> card. The card's BMC continues to support features such as power sequence
> management, board monitoring via sensors, JTAG management and in-band SPI
> interface access. The external Flash on the card shall be programmed with the
> Intel root public key hash (for BMC images and FW) and Customer/User key (for
> FPGA Flat image) during manufacturing, so the image updates (RSU) shall be
> verified using ECDSA-256 P-256 and SHA2-256 before being written to the
> MAX10 Image Flash or FPGA Image Flash.
> >
> > This patch add RoT support for MAX10 because some registers and the content
> of Device Tree have changed between RoT solution and Non-RoT solution.
> 
> Thanks Tianfei for the additional information, I will update commit as following,
> please let me know if you have any objection:
> 
> raw/ifpga/base: support max10 security feature
> 
> In PAC N3000 Card, MAX10 Board Management Controller (BMC) implements
> the security functionality.
> 
> Security functionality adds secure Remote System Update (RSU) authentication
> and integrity checks for FPGA flat image, and FW updates to the card.
> 
> This patch adds security feature support for MAX10, in secure solution some
> registers and the content of the Device Tree changes.

It looks good for me, thanks a lot!

> 
> >
> >
> >>
> >> Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
> >> Signed-off-by: Andy Pei <andy.pei@intel.com>
> >> ---
> >>  drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
> >>  drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
> >>  drivers/raw/ifpga/base/opae_intel_max10.c | 137
> >> +++++++++++++++++++++++++-----
> >> drivers/raw/ifpga/base/opae_intel_max10.h |  80 ++++++++++++-----
> >>  4 files changed, 198 insertions(+), 47 deletions(-)
> >>
> >> diff --git a/drivers/raw/ifpga/base/ifpga_defines.h
> >> b/drivers/raw/ifpga/base/ifpga_defines.h
> >> index 8993cc6..1e84b15 100644
> >> --- a/drivers/raw/ifpga/base/ifpga_defines.h
> >> +++ b/drivers/raw/ifpga/base/ifpga_defines.h
> >> @@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
> >>  	u32 patch_version;
> >>  	u32 minor_version;
> >>  	u32 major_version;
> >> +	u32 max10_version;
> >> +	u32 nios_fw_version;
> >>  	u32 nums_of_retimer;
> >>  	u32 ports_per_retimer;
> >>  	u32 nums_of_fvl;
> >> diff --git a/drivers/raw/ifpga/base/ifpga_fme.c
> >> b/drivers/raw/ifpga/base/ifpga_fme.c
> >> index 794ca09..87fa596 100644
> >> --- a/drivers/raw/ifpga/base/ifpga_fme.c
> >> +++ b/drivers/raw/ifpga/base/ifpga_fme.c
> >> @@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,  static
> >> int fme_get_board_interface(struct ifpga_fme_hw *fme)  {
> >>  	struct fme_bitstream_id id;
> >> +	u32 val;
> >>
> >>  	if (fme_hdr_get_bitstream_id(fme, &id.id))
> >>  		return -EINVAL;
> >> @@ -850,6 +851,18 @@ static int fme_get_board_interface(struct
> >> ifpga_fme_hw *fme)
> >>  			fme->board_info.nums_of_fvl,
> >>  			fme->board_info.ports_per_fvl);
> >>
> >> +	if (max10_sys_read(MAX10_BUILD_VER, &val))
> >> +		return -EINVAL;
> >> +	fme->board_info.max10_version = val & 0xffffff;
> >> +
> >> +	if (max10_sys_read(NIOS2_FW_VERSION, &val))
> >> +		return -EINVAL;
> >> +	fme->board_info.nios_fw_version = val & 0xffffff;
> >> +
> >> +	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
> >> +		fme->board_info.max10_version,
> >> +		fme->board_info.nios_fw_version);
> >> +
> >>  	return 0;
> >>  }
> >>
> >> @@ -858,16 +871,11 @@ static int spi_self_checking(void)
> >>  	u32 val;
> >>  	int ret;
> >>
> >> -	ret = max10_reg_read(0x30043c, &val);
> >> +	ret = max10_sys_read(MAX10_TEST_REG, &val);
> >>  	if (ret)
> >>  		return -EIO;
> >>
> >> -	if (val != 0x87654321) {
> >> -		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
> >> -		return -EIO;
> >> -	}
> >> -
> >> -	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
> >> +	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
> >>
> >>  	return 0;
> >>  }
> >> @@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct
> >> ifpga_fme_hw *fme,
> >>  	if (!dev)
> >>  		return -ENODEV;
> >>
> >> -	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
> >> +	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
> >>  		dev_err(dev, "%s: read pkvl status fail\n", __func__);
> >>  		return -EINVAL;
> >>  	}
> >> @@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct
> >> ifpga_fme_hw *fme,
> >>  	if (!dev)
> >>  		return -ENODEV;
> >>
> >> -	if (max10_reg_read(sensor->value_reg, value)) {
> >> +	if (max10_sys_read(sensor->value_reg, value)) {
> >>  		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
> >>  				__func__, sensor->value_reg);
> >>  		return -EINVAL;
> >> diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c
> >> b/drivers/raw/ifpga/base/opae_intel_max10.c
> >> index ae7a8df..748ab56 100644
> >> --- a/drivers/raw/ifpga/base/opae_intel_max10.c
> >> +++ b/drivers/raw/ifpga/base/opae_intel_max10.c
> >> @@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int
> val)
> >>  			reg, 4, (unsigned char *)&tmp);
> >>  }
> >>
> >> +int max10_sys_read(unsigned int offset, unsigned int *val) {
> >> +	if (!g_max10)
> >> +		return -ENODEV;
> >> +
> >> +	return max10_reg_read(g_max10->base + offset, val); }
> >> +
> >> +int max10_sys_write(unsigned int offset, unsigned int val) {
> >> +	if (!g_max10)
> >> +		return -ENODEV;
> >> +
> >> +	return max10_reg_write(g_max10->base + offset, val); }
> >> +
> >>  static struct max10_compatible_id max10_id_table[] = {
> >>  	{.compatible = MAX10_PAC,},
> >>  	{.compatible = MAX10_PAC_N3000,},
> >> @@ -66,7 +82,8 @@ static void max10_check_capability(struct
> >> intel_max10_device *max10)
> >>  		max10->flags |= MAX10_FLAGS_NO_I2C2 |
> >>  				MAX10_FLAGS_NO_BMCIMG_FLASH;
> >>  		dev_info(max10, "found %s card\n", max10->id->compatible);
> >> -	}
> >> +	} else
> >> +		max10->flags |= MAX10_FLAGS_MAC_CACHE;
> >>  }
> >>
> >>  static int altera_nor_flash_read(u32 offset, @@ -100,7 +117,7 @@
> >> static int enable_nor_flash(bool on)
> >>  	unsigned int val = 0;
> >>  	int ret;
> >>
> >> -	ret = max10_reg_read(RSU_REG_OFF, &val);
> >> +	ret = max10_sys_read(RSU_REG, &val);
> >>  	if (ret) {
> >>  		dev_err(NULL "enabling flash error\n");
> >>  		return ret;
> >> @@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
> >>  	else
> >>  		val &= ~RSU_ENABLE;
> >>
> >> -	return max10_reg_write(RSU_REG_OFF, val);
> >> +	return max10_sys_write(RSU_REG, val);
> >>  }
> >>
> >>  static int init_max10_device_table(struct intel_max10_device *max10)
> >> @@
> >> -123,7 +140,7 @@ static int init_max10_device_table(struct
> >> intel_max10_device *max10)
> >>  	u32 dt_size, dt_addr, val;
> >>  	int ret;
> >>
> >> -	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
> >> +	ret = max10_sys_read(DT_AVAIL_REG, &val);
> >>  	if (ret) {
> >>  		dev_err(max10 "cannot read DT_AVAIL_REG\n");
> >>  		return ret;
> >> @@ -134,7 +151,7 @@ static int init_max10_device_table(struct
> >> intel_max10_device *max10)
> >>  		return -EINVAL;
> >>  	}
> >>
> >> -	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
> >> +	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
> >>  	if (ret) {
> >>  		dev_info(max10 "cannot get base addr of device table\n");
> >>  		return ret;
> >> @@ -315,7 +332,7 @@ static int max10_add_sensor(struct
> >> raw_sensor_info *info,
> >>  		if (!sensor_reg_valid(&info->regs[i]))
> >>  			continue;
> >>
> >> -		ret = max10_reg_read(info->regs[i].regoff, &val);
> >> +		ret = max10_sys_read(info->regs[i].regoff, &val);
> >>  		if (ret)
> >>  			break;
> >>
> >> @@ -355,7 +372,8 @@ static int max10_add_sensor(struct
> >> raw_sensor_info *info,
> >>  	return ret;
> >>  }
> >>
> >> -static int max10_sensor_init(struct intel_max10_device *dev)
> >> +static int
> >> +max10_sensor_init(struct intel_max10_device *dev, int parent)
> >>  {
> >>  	int i, ret = 0, offset = 0;
> >>  	const fdt32_t *num;
> >> @@ -370,7 +388,7 @@ static int max10_sensor_init(struct
> >> intel_max10_device
> >> *dev)
> >>  		return 0;
> >>  	}
> >>
> >> -	fdt_for_each_subnode(offset, fdt_root, 0) {
> >> +	fdt_for_each_subnode(offset, fdt_root, parent) {
> >>  		ptr = fdt_get_name(fdt_root, offset, NULL);
> >>  		if (!ptr) {
> >>  			dev_err(dev, "failed to fdt get name\n"); @@ -417,7
> +435,16 @@
> >> static int max10_sensor_init(struct intel_max10_device *dev)
> >>  				continue;
> >>  			}
> >>
> >> -			raw->regs[i].regoff = start;
> >> +			/* This is a hack to compatible with non-secure
> >> +			 * solution. If sensors are included in root node,
> >> +			 * then it's non-secure dtb, which use absolute addr
> >> +			 * of non-secure solution.
> >> +			 */
> >> +			if (parent)
> >> +				raw->regs[i].regoff = start;
> >> +			else
> >> +				raw->regs[i].regoff = start -
> >> +					MAX10_BASE_ADDR;
> >>  			raw->regs[i].size = size;
> >>  		}
> >>
> >> @@ -469,6 +496,63 @@ static int max10_sensor_init(struct
> >> intel_max10_device *dev)
> >>  	return ret;
> >>  }
> >>
> >> +static int check_max10_version(struct intel_max10_device *dev) {
> >> +	unsigned int v;
> >> +
> >> +	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
> >> +				&v)) {
> >> +		if (v != 0xffffffff) {
> >> +			dev_info(dev, "secure MAX10 detected\n");
> >> +			dev->base = MAX10_SEC_BASE_ADDR;
> >> +			dev->flags |= MAX10_FLAGS_SECURE;
> >> +		} else {
> >> +			dev_info(dev, "non-secure MAX10 detected\n");
> >> +			dev->base = MAX10_BASE_ADDR;
> >> +		}
> >> +		return 0;
> >> +	}
> >> +
> >> +	return -ENODEV;
> >> +}
> >> +
> >> +static int
> >> +max10_secure_hw_init(struct intel_max10_device *dev) {
> >> +	int offset, sysmgr_offset = 0;
> >> +	char *fdt_root;
> >> +
> >> +	fdt_root = dev->fdt_root;
> >> +	if (!fdt_root) {
> >> +		dev_debug(dev, "skip init as not find Device Tree\n");
> >> +		return 0;
> >> +	}
> >> +
> >> +	fdt_for_each_subnode(offset, fdt_root, 0) {
> >> +		if (!fdt_node_check_compatible(fdt_root, offset,
> >> +					"intel-max10,system-manager")) {
> >> +			sysmgr_offset = offset;
> >> +			break;
> >> +		}
> >> +	}
> >> +
> >> +	max10_check_capability(dev);
> >> +
> >> +	max10_sensor_init(dev, sysmgr_offset);
> >> +
> >> +	return 0;
> >> +}
> >> +
> >> +static int
> >> +max10_non_secure_hw_init(struct intel_max10_device *dev) {
> >> +	max10_check_capability(dev);
> >> +
> >> +	max10_sensor_init(dev, 0);
> >> +
> >> +	return 0;
> >> +}
> >> +
> >>  struct intel_max10_device *
> >>  intel_max10_device_probe(struct altera_spi_device *spi,
> >>  		int chipselect)
> >> @@ -492,32 +576,47 @@ struct intel_max10_device *
> >>  	/* set the max10 device firstly */
> >>  	g_max10 = dev;
> >>
> >> -	/* init the MAX10 device table */
> >> +	/* check the max10 version */
> >> +	ret = check_max10_version(dev);
> >> +	if (ret) {
> >> +		dev_err(dev, "Failed to find max10 hardware!\n");
> >> +		goto free_dev;
> >> +	}
> >> +
> >> +	/* load the MAX10 device table */
> >>  	ret = init_max10_device_table(dev);
> >>  	if (ret) {
> >> -		dev_err(dev, "init max10 device table fail\n");
> >> +		dev_err(dev, "Init max10 device table fail\n");
> >>  		goto free_dev;
> >>  	}
> >>
> >> -	max10_check_capability(dev);
> >> +	/* init max10 devices, like sensor*/
> >> +	if (dev->flags & MAX10_FLAGS_SECURE)
> >> +		ret = max10_secure_hw_init(dev);
> >> +	else
> >> +		ret = max10_non_secure_hw_init(dev);
> >> +	if (ret) {
> >> +		dev_err(dev, "Failed to init max10 hardware!\n");
> >> +		goto free_dtb;
> >> +	}
> >>
> >>  	/* read FPGA loading information */
> >> -	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
> >> +	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
> >>  	if (ret) {
> >>  		dev_err(dev, "fail to get FPGA loading info\n");
> >> -		goto spi_tran_fail;
> >> +		goto release_max10_hw;
> >>  	}
> >>  	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" :
> >> "Factory");
> >>
> >> -
> >> -	max10_sensor_init(dev);
> >> -
> >>  	return dev;
> >>
> >> -spi_tran_fail:
> >> +release_max10_hw:
> >> +	max10_sensor_uinit();
> >> +free_dtb:
> >>  	if (dev->fdt_root)
> >>  		opae_free(dev->fdt_root);
> >> -	spi_transaction_remove(dev->spi_tran_dev);
> >> +	if (dev->spi_tran_dev)
> >> +		spi_transaction_remove(dev->spi_tran_dev);
> >>  free_dev:
> >>  	g_max10 = NULL;
> >>  	opae_free(dev);
> >> diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h
> >> b/drivers/raw/ifpga/base/opae_intel_max10.h
> >> index 90bf098..e632941 100644
> >> --- a/drivers/raw/ifpga/base/opae_intel_max10.h
> >> +++ b/drivers/raw/ifpga/base/opae_intel_max10.h
> >> @@ -23,6 +23,8 @@ struct max10_compatible_id {
> >>  #define MAX10_FLAGS_SPI                 BIT(3)
> >>  #define MAX10_FLGAS_NIOS_SPI            BIT(4)
> >>  #define MAX10_FLAGS_PKVL                BIT(5)
> >> +#define MAX10_FLAGS_SECURE		BIT(6)
> >> +#define MAX10_FLAGS_MAC_CACHE		BIT(7)
> >>
> >>  struct intel_max10_device {
> >>  	unsigned int flags; /*max10 hardware capability*/ @@ -30,6 +32,7
> @@
> >> struct intel_max10_device {
> >>  	struct spi_transaction_dev *spi_tran_dev;
> >>  	struct max10_compatible_id *id; /*max10 compatible*/
> >>  	char *fdt_root;
> >> +	unsigned int base; /* max10 base address */
> >>  };
> >>
> >>  /* retimer speed */
> >> @@ -74,30 +77,69 @@ struct opae_retimer_status {  #define FLASH_BASE
> >> 0x10000000  #define FLASH_OPTION_BITS 0x10000
> >>
> >> -#define NIOS2_FW_VERSION_OFF   0x300400
> >> -#define RSU_REG_OFF            0x30042c
> >> -#define FPGA_RP_LOAD		BIT(3)
> >> -#define NIOS2_PRERESET		BIT(4)
> >> -#define NIOS2_HANG		BIT(5)
> >> -#define RSU_ENABLE		BIT(6)
> >> -#define NIOS2_RESET		BIT(7)
> >> -#define NIOS2_I2C2_POLL_STOP	BIT(13)
> >> -#define FPGA_RECONF_REG_OFF	0x300430
> >> -#define COUNTDOWN_START		BIT(18)
> >> -#define MAX10_BUILD_VER_OFF	0x300468
> >> -#define PCB_INFO		GENMASK(31, 24)
> >> -#define MAX10_BUILD_VERION	GENMASK(23, 0)
> >> -#define FPGA_PAGE_INFO_OFF	0x30046c
> >> -#define DT_AVAIL_REG_OFF	0x300490
> >> -#define DT_AVAIL		BIT(0)
> >> -#define DT_BASE_ADDR_REG_OFF	0x300494
> >> -#define PKVL_POLLING_CTRL       0x300480
> >> -#define PKVL_LINK_STATUS        0x300564
> >> +/* System Registers */
> >> +#define MAX10_BASE_ADDR		0x300400
> >> +#define MAX10_SEC_BASE_ADDR	0x300800
> >> +/* Register offset of system registers */
> >> +#define NIOS2_FW_VERSION	0x0
> >> +#define MAX10_MACADDR1		0x10
> >> +#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
> >> +#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
> >> +#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
> >> +#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
> >> +#define MAX10_MACADDR2		0x14
> >> +#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
> >> +#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
> >> +#define   MAX10_MAC_COUNT	GENMASK(23, 16)
> >> +#define RSU_REG			0x2c
> >> +#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
> >> +#define   FPGA_RP_LOAD		BIT(3)
> >> +#define   NIOS2_PRERESET	BIT(4)
> >> +#define   NIOS2_HANG		BIT(5)
> >> +#define   RSU_ENABLE		BIT(6)
> >> +#define   NIOS2_RESET		BIT(7)
> >> +#define   NIOS2_I2C2_POLL_STOP	BIT(13)
> >> +#define   PKVL_EEPROM_LOAD	BIT(31)
> >> +#define FPGA_RECONF_REG		0x30
> >> +#define MAX10_TEST_REG		0x3c
> >> +#define   COUNTDOWN_START	BIT(18)
> >> +#define MAX10_BUILD_VER		0x68
> >> +#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
> >> +#define   PCB_INFO		GENMASK(31, 24)
> >> +#define FPGA_PAGE_INFO		0x6c
> >> +#define DT_AVAIL_REG		0x90
> >> +#define   DT_AVAIL		BIT(0)
> >> +#define DT_BASE_ADDR_REG	0x94
> >> +#define MAX10_DOORBELL		0x400
> >> +#define   RSU_REQUEST		BIT(0)
> >> +#define   SEC_PROGRESS		GENMASK(7, 4)
> >> +#define   HOST_STATUS		GENMASK(11, 8)
> >> +#define   SEC_STATUS		GENMASK(23, 16)
> >> +
> >> +/* PKVL related registers, in system register region */
> >> +#define PKVL_POLLING_CTRL		0x80
> >> +#define   POLLING_MODE			GENMASK(15, 0)
> >> +#define   PKVL_A_PRELOAD		BIT(16)
> >> +#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
> >> +#define   PKVL_A_DATA_TOO_BIG		BIT(18)
> >> +#define   PKVL_A_HDR_CHECKSUM		BIT(20)
> >> +#define   PKVL_B_PRELOAD		BIT(24)
> >> +#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
> >> +#define   PKVL_B_DATA_TOO_BIG		BIT(26)
> >> +#define   PKVL_B_HDR_CHECKSUM		BIT(28)
> >> +#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
> >> +#define PKVL_LINK_STATUS		0x164
> >> +#define PKVL_A_VERSION			0x254
> >> +#define PKVL_B_VERSION			0x258
> >> +#define   SERDES_VERSION		GENMASK(15, 0)
> >> +#define   SBUS_VERSION			GENMASK(31, 16)
> >>
> >>  #define DFT_MAX_SIZE		0x7e0000
> >>
> >>  int max10_reg_read(unsigned int reg, unsigned int *val);  int
> >> max10_reg_write(unsigned int reg, unsigned int val);
> >> +int max10_sys_read(unsigned int offset, unsigned int *val); int
> >> +max10_sys_write(unsigned int offset, unsigned int val);
> >>  struct intel_max10_device *
> >>  intel_max10_device_probe(struct altera_spi_device *spi,
> >>  		int chipselect);
> >> --
> >> 1.8.3.1
> >

[dpdk-dev] [PATCH v18 14/19] raw/ifpga/base: configure FEC mode

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 15/19] raw/ifpga/base: clean fme errors

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occurred.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 5d6d630..5905eac 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index c6c69c2..c7bf085 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1175,6 +1175,25 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1184,6 +1203,9 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 16/19] raw/ifpga/base: add new API get board info

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..9f0147d 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disaggregate;
+	u8 lightweight;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..799d67d 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweight = id.lightweiht;
+	fme->board_info.disaggregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweight:%s\n"
+			"disaggregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweight),
+			check_support(fme->board_info.disaggregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 17/19] raw/ifpga: add lightweight fpga image support

[Rosen Xu]

From: Andy Pei <andy.pei@intel.com>

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index c7bf085..cfcbf68 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -836,6 +836,8 @@ static int set_surprise_link_check_aer(
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -862,22 +864,40 @@ static int set_surprise_link_check_aer(
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweight) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
-		uuid.b + 8, sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 18/19] raw/ifpga/base: add multiple cards support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, there is one MAX10 chip in each card, and
all of the sensors are connected to MAX10 chip. To support multiple
cards in one server, we introducing a sensor device list under
intel_max10_device instead of a global list. On the other hand, we
using separate intel_max10_device instance for each opae_adatper.

Add mutex lock on do_transaction() function for SPI driver to avoid
race condition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c            |  40 +++++++---
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          |  14 ++--
 drivers/raw/ifpga/base/opae_hw_api.h          |  15 ++--
 drivers/raw/ifpga/base/opae_i2c.c             |  44 ++++++++---
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_intel_max10.c     | 110 ++++++++++++++------------
 drivers/raw/ifpga/base/opae_intel_max10.h     |  19 +++--
 drivers/raw/ifpga/base/opae_spi.c             |   5 --
 drivers/raw/ifpga/base/opae_spi.h             |   3 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  44 +++++++++--
 drivers/raw/ifpga/ifpga_rawdev.c              |  14 ++--
 12 files changed, 205 insertions(+), 109 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 799d67d..c31a94c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -839,8 +839,13 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	struct ifpga_hw *hw;
 	u32 val;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
@@ -854,7 +859,10 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	fme->board_info.seu = id.seu;
 	fme->board_info.ptp = id.ptp;
 
-	dev_info(fme, "found: board: %s type: %s\n",
+	dev_info(fme, "found: PCI dev: %02x:%02x:%x board: %s type: %s\n",
+			hw->pci_data->bus,
+			hw->pci_data->devid,
+			hw->pci_data->function,
 			board_major_to_string(fme->board_info.major),
 			board_type_to_string(fme->board_info.type));
 
@@ -882,11 +890,11 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
-	if (max10_sys_read(MAX10_BUILD_VER, &val))
+	if (max10_sys_read(fme->max10_dev, MAX10_BUILD_VER, &val))
 		return -EINVAL;
 	fme->board_info.max10_version = val & 0xffffff;
 
-	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+	if (max10_sys_read(fme->max10_dev, NIOS2_FW_VERSION, &val))
 		return -EINVAL;
 	fme->board_info.nios_fw_version = val & 0xffffff;
 
@@ -897,12 +905,12 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	return 0;
 }
 
-static int spi_self_checking(void)
+static int spi_self_checking(struct intel_max10_device *dev)
 {
 	u32 val;
 	int ret;
 
-	ret = max10_sys_read(MAX10_TEST_REG, &val);
+	ret = max10_sys_read(dev, MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
@@ -937,10 +945,11 @@ static int fme_spi_init(struct ifpga_feature *feature)
 		goto spi_fail;
 	}
 
+
 	fme->max10_dev = max10;
 
 	/* SPI self test */
-	if (spi_self_checking()) {
+	if (spi_self_checking(max10)) {
 		ret = -EIO;
 		goto max10_fail;
 	}
@@ -1041,8 +1050,18 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 	struct ifpga_fme_hw *fme = (struct ifpga_fme_hw *)feature->parent;
 	struct altera_spi_device *spi_master;
 	struct intel_max10_device *max10;
+	struct ifpga_hw *hw;
+	struct opae_manager *mgr;
 	int ret = 0;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
+	mgr = hw->adapter->mgr;
+	if (!mgr)
+		return -ENODEV;
+
 	dev_info(fme, "FME SPI Master (NIOS) Init.\n");
 	dev_debug(fme, "FME SPI base addr %p.\n",
 			feature->addr);
@@ -1080,12 +1099,15 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 		goto release_dev;
 	}
 
+	max10->bus = hw->pci_data->bus;
+
 	fme_get_board_interface(fme);
 
 	fme->max10_dev = max10;
+	mgr->sensor_list = &max10->opae_sensor_list;
 
 	/* SPI self test */
-	if (spi_self_checking())
+	if (spi_self_checking(max10))
 		goto spi_fail;
 
 	return ret;
@@ -1344,7 +1366,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(dev, PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1372,7 +1394,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(sensor->value_reg, value)) {
+	if (max10_sys_read(dev, sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_debug.c b/drivers/raw/ifpga/base/opae_debug.c
index 88f2d5c..dad3ea3 100644
--- a/drivers/raw/ifpga/base/opae_debug.c
+++ b/drivers/raw/ifpga/base/opae_debug.c
@@ -59,6 +59,9 @@ static void opae_adapter_data_dump(void *data)
 		opae_log("OPAE Adapter Type = PCI\n");
 		opae_log("PCI Device ID: 0x%04x\n", d_pci->device_id);
 		opae_log("PCI Vendor ID: 0x%04x\n", d_pci->vendor_id);
+		opae_log("PCI bus: 0x%04x\n", d_pci->bus);
+		opae_log("PCI devid: 0x%04x\n", d_pci->devid);
+		opae_log("PCI function: 0x%04x\n", d_pci->function);
 
 		for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 			r = &d_pci->region[i];
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 1ccc967..c969dfe 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -583,11 +583,12 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_id(unsigned int id)
+opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (sensor->id == id)
 			return sensor;
 
@@ -601,11 +602,12 @@ struct opae_sensor_info *
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_name(const char *name)
+opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (!strcmp(sensor->name, name))
 			return sensor;
 
@@ -630,7 +632,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_name(name);
+	sensor = opae_mgr_get_sensor_by_name(mgr, name);
 	if (!sensor)
 		return -ENODEV;
 
@@ -658,7 +660,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_id(id);
+	sensor = opae_mgr_get_sensor_by_id(mgr, id);
 	if (!sensor)
 		return -ENODEV;
 
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index b78fbd5..cdf369f 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -40,6 +40,7 @@ struct opae_manager {
 	struct opae_adapter *adapter;
 	struct opae_manager_ops *ops;
 	struct opae_manager_networking_ops *network_ops;
+	struct opae_sensor_list *sensor_list;
 	void *data;
 };
 
@@ -75,9 +76,8 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
-extern struct opae_sensor_list opae_sensor_list;
-#define opae_mgr_for_each_sensor(sensor) \
-	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+#define opae_mgr_for_each_sensor(mgr, sensor) \
+	TAILQ_FOREACH(sensor, mgr->sensor_list, node)
 
 /* OPAE Manager APIs */
 struct opae_manager *
@@ -88,8 +88,10 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
-struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
-struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id);
 int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
 		const char *name, unsigned int *value);
 int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
@@ -241,6 +243,9 @@ struct opae_adapter_data_pci {
 	enum opae_adapter_type type;
 	u16 device_id;
 	u16 vendor_id;
+	u16 bus; /*Device bus for PCI */
+	u16 devid; /* Device ID */
+	u16 function; /* Device function */
 	struct opae_reg_region region[PCI_MAX_RESOURCE];
 	int vfio_dev_fd;  /* VFIO device file descriptor */
 };
diff --git a/drivers/raw/ifpga/base/opae_i2c.c b/drivers/raw/ifpga/base/opae_i2c.c
index f8bc247..846d751 100644
--- a/drivers/raw/ifpga/base/opae_i2c.c
+++ b/drivers/raw/ifpga/base/opae_i2c.c
@@ -28,6 +28,9 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 {
 	u8 msgbuf[2];
 	int i = 0;
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
 
 	if (flags & I2C_FLAG_ADDR16)
 		msgbuf[i++] = offset >> 8;
@@ -49,10 +52,16 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		},
 	};
 
-	if (!dev->xfer)
-		return -ENODEV;
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	ret = i2c_transfer(dev, msg, 2);
 
-	return i2c_transfer(dev, msg, 2);
+exit:
+	pthread_mutex_unlock(&dev->lock);
+	return ret;
 }
 
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
@@ -63,12 +72,18 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	int ret;
 	int i = 0;
 
-	if (!dev->xfer)
-		return -ENODEV;
+	pthread_mutex_lock(&dev->lock);
+
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
 
 	buf = opae_malloc(I2C_MAX_OFFSET_LEN + len);
-	if (!buf)
-		return -ENOMEM;
+	if (!buf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
 
 	msg.addr = slave_addr;
 	msg.flags = 0;
@@ -84,6 +99,8 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	ret = i2c_transfer(dev, &msg, 1);
 
 	opae_free(buf);
+exit:
+	pthread_mutex_unlock(&dev->lock);
 	return ret;
 }
 
@@ -477,14 +494,19 @@ struct altera_i2c_dev *altera_i2c_probe(void *base)
 	dev->i2c_clk = dev->i2c_param.ref_clk * 1000000;
 	dev->xfer = altera_i2c_xfer;
 
+	if (pthread_mutex_init(&dev->lock, NULL))
+		return NULL;
+
 	altera_i2c_hardware_init(dev);
 
 	return dev;
 }
 
-int altera_i2c_remove(struct altera_i2c_dev *dev)
+void altera_i2c_remove(struct altera_i2c_dev *dev)
 {
-	altera_i2c_disable(dev);
-
-	return 0;
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
+		altera_i2c_disable(dev);
+		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/base/opae_i2c.h b/drivers/raw/ifpga/base/opae_i2c.h
index 8890c8f..266e127 100644
--- a/drivers/raw/ifpga/base/opae_i2c.h
+++ b/drivers/raw/ifpga/base/opae_i2c.h
@@ -93,6 +93,7 @@ struct altera_i2c_dev {
 	u32 isr_mask;
 	u8 *buf;
 	int (*xfer)(struct altera_i2c_dev *dev, struct i2c_msg *msg, int num);
+	pthread_mutex_t lock;
 };
 
 /**
@@ -114,7 +115,7 @@ enum i2c_msg_flags {
 };
 
 struct altera_i2c_dev *altera_i2c_probe(void *base);
-int altera_i2c_remove(struct altera_i2c_dev *dev);
+void altera_i2c_remove(struct altera_i2c_dev *dev);
 int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		u32 offset, u8 *buf, u32 count);
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 748ab56..8e23ca1 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -5,45 +5,50 @@
 #include "opae_intel_max10.h"
 #include <libfdt.h>
 
-static struct intel_max10_device *g_max10;
-
-struct opae_sensor_list opae_sensor_list =
-	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
-
-int max10_reg_read(unsigned int reg, unsigned int *val)
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_read(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x\n", __func__, dev->bus, reg);
+
+	return spi_transaction_read(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)val);
 }
 
-int max10_reg_write(unsigned int reg, unsigned int val)
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val)
 {
 	unsigned int tmp = val;
 
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_write(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x, val:0x%x\n", __func__,
+			dev->bus, reg, val);
+
+	return spi_transaction_write(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)&tmp);
 }
 
-int max10_sys_read(unsigned int offset, unsigned int *val)
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_read(g_max10->base + offset, val);
+
+	return max10_reg_read(dev, dev->base + offset, val);
 }
 
-int max10_sys_write(unsigned int offset, unsigned int val)
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_write(g_max10->base + offset, val);
+	return max10_reg_write(dev, dev->base + offset, val);
 }
 
 static struct max10_compatible_id max10_id_table[] = {
@@ -86,8 +91,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
-static int altera_nor_flash_read(u32 offset,
-		void *buffer, u32 len)
+static int altera_nor_flash_read(struct intel_max10_device *dev,
+	u32 offset, void *buffer, u32 len)
 {
 	int word_len;
 	int i;
@@ -95,13 +100,13 @@ static int altera_nor_flash_read(u32 offset,
 	unsigned int value;
 	int ret;
 
-	if (!buffer || len <= 0)
+	if (!dev || !buffer || len <= 0)
 		return -ENODEV;
 
 	word_len = len/4;
 
 	for (i = 0; i < word_len; i++) {
-		ret = max10_reg_read(offset + i*4,
+		ret = max10_reg_read(dev, offset + i*4,
 				&value);
 		if (ret)
 			return -EBUSY;
@@ -112,12 +117,12 @@ static int altera_nor_flash_read(u32 offset,
 	return 0;
 }
 
-static int enable_nor_flash(bool on)
+static int enable_nor_flash(struct intel_max10_device *dev, bool on)
 {
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_sys_read(RSU_REG, &val);
+	ret = max10_sys_read(dev, RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -128,7 +133,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_sys_write(RSU_REG, val);
+	return max10_sys_write(dev, RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -140,7 +145,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_sys_read(DT_AVAIL_REG, &val);
+	ret = max10_sys_read(max10, DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -151,19 +156,19 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
+	ret = max10_sys_read(max10, DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
 	}
 
-	ret = enable_nor_flash(true);
+	ret = enable_nor_flash(max10, true);
 	if (ret) {
 		dev_err(max10 "fail to enable flash\n");
 		return ret;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	ret = altera_nor_flash_read(max10, dt_addr, &hdr, sizeof(hdr));
 	if (ret) {
 		dev_err(max10 "read fdt header fail\n");
 		goto done;
@@ -188,7 +193,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		goto done;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	ret = altera_nor_flash_read(max10, dt_addr, fdt_root, dt_size);
 	if (ret) {
 		dev_err(max10 "cannot read device table\n");
 		goto done;
@@ -207,7 +212,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	max10->fdt_root = fdt_root;
 
 done:
-	ret = enable_nor_flash(false);
+	ret = enable_nor_flash(max10, false);
 
 	if (ret && fdt_root)
 		opae_free(fdt_root);
@@ -298,12 +303,12 @@ static int fdt_get_named_reg(const void *fdt, int node, const char *name,
 	return fdt_get_reg(fdt, node, idx, start, size);
 }
 
-static void max10_sensor_uinit(void)
+static void max10_sensor_uinit(struct intel_max10_device *dev)
 {
 	struct opae_sensor_info *info;
 
-	TAILQ_FOREACH(info, &opae_sensor_list, node) {
-		TAILQ_REMOVE(&opae_sensor_list, info, node);
+	TAILQ_FOREACH(info, &dev->opae_sensor_list, node) {
+		TAILQ_REMOVE(&dev->opae_sensor_list, info, node);
 		opae_free(info);
 	}
 }
@@ -313,8 +318,8 @@ static bool sensor_reg_valid(struct sensor_reg *reg)
 	return !!reg->size;
 }
 
-static int max10_add_sensor(struct raw_sensor_info *info,
-		struct opae_sensor_info *sensor)
+static int max10_add_sensor(struct intel_max10_device *dev,
+	struct raw_sensor_info *info, struct opae_sensor_info *sensor)
 {
 	int i;
 	int ret = 0;
@@ -332,12 +337,15 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_sys_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(dev, info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
-		if (val == 0xdeadbeef)
+		if (val == 0xdeadbeef) {
+			dev_debug(dev, "%s: sensor:%s invalid 0x%x at:%d\n",
+				__func__, sensor->name, val, i);
 			continue;
+		}
 
 		val *= info->multiplier;
 
@@ -458,7 +466,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
 		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
 
-		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+		dev_debug(dev, "found sensor from DTB: %s: %s: %u: %u\n",
 				raw->name, raw->type,
 				raw->id, raw->multiplier);
 
@@ -473,15 +481,16 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 			goto free_sensor;
 		}
 
-		if (max10_add_sensor(raw, sensor)) {
+		if (max10_add_sensor(dev, raw, sensor)) {
 			ret = -EINVAL;
 			opae_free(sensor);
 			goto free_sensor;
 		}
 
-		if (sensor->flags & OPAE_SENSOR_VALID)
-			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
-		else
+		if (sensor->flags & OPAE_SENSOR_VALID) {
+			TAILQ_INSERT_TAIL(&dev->opae_sensor_list, sensor, node);
+			dev_info(dev, "found valid sensor: %s\n", sensor->name);
+		} else
 			opae_free(sensor);
 
 		opae_free(raw);
@@ -492,7 +501,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 free_sensor:
 	if (raw)
 		opae_free(raw);
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 	return ret;
 }
 
@@ -500,7 +509,7 @@ static int check_max10_version(struct intel_max10_device *dev)
 {
 	unsigned int v;
 
-	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+	if (!max10_reg_read(dev, MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
 				&v)) {
 		if (v != 0xffffffff) {
 			dev_info(dev, "secure MAX10 detected\n");
@@ -565,6 +574,8 @@ struct intel_max10_device *
 	if (!dev)
 		return NULL;
 
+	TAILQ_INIT(&dev->opae_sensor_list);
+
 	dev->spi_master = spi;
 
 	dev->spi_tran_dev = spi_transaction_init(spi, chipselect);
@@ -573,9 +584,6 @@ struct intel_max10_device *
 		goto free_dev;
 	}
 
-	/* set the max10 device firstly */
-	g_max10 = dev;
-
 	/* check the max10 version */
 	ret = check_max10_version(dev);
 	if (ret) {
@@ -601,7 +609,7 @@ struct intel_max10_device *
 	}
 
 	/* read FPGA loading information */
-	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
+	ret = max10_sys_read(dev, FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
 		goto release_max10_hw;
@@ -611,14 +619,13 @@ struct intel_max10_device *
 	return dev;
 
 release_max10_hw:
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return NULL;
@@ -629,7 +636,7 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
@@ -637,7 +644,6 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index e632941..123cdc4 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -26,6 +26,9 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SECURE		BIT(6)
 #define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
@@ -33,6 +36,8 @@ struct intel_max10_device {
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
 	unsigned int base; /* max10 base address */
+	u16 bus;
+	struct opae_sensor_list opae_sensor_list;
 };
 
 /* retimer speed */
@@ -136,17 +141,19 @@ struct opae_retimer_status {
 
 #define DFT_MAX_SIZE		0x7e0000
 
-int max10_reg_read(unsigned int reg, unsigned int *val);
-int max10_reg_write(unsigned int reg, unsigned int val);
-int max10_sys_read(unsigned int offset, unsigned int *val);
-int max10_sys_write(unsigned int offset, unsigned int val);
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val);
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val);
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val);
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
-/** List of opae sensors */
-TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
 
 #define SENSOR_REG_VALUE 0x0
 #define SENSOR_REG_HIGH_WARN 0x1
diff --git a/drivers/raw/ifpga/base/opae_spi.c b/drivers/raw/ifpga/base/opae_spi.c
index cc52782..bfdc83e 100644
--- a/drivers/raw/ifpga/base/opae_spi.c
+++ b/drivers/raw/ifpga/base/opae_spi.c
@@ -181,7 +181,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 	u32 rxd;
 	unsigned int tx_data;
 	u32 status;
-	int retry = 0;
 	int ret;
 
 	while (count < dev->len) {
@@ -194,10 +193,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 				return -EIO;
 			if (status & ALTERA_SPI_STATUS_RRDY_MSK)
 				break;
-			if (retry++ > SPI_MAX_RETRY) {
-				dev_err(dev, "%s, read timeout\n", __func__);
-				return -EBUSY;
-			}
 		}
 
 		ret = spi_reg_read(dev, ALTERA_SPI_RXDATA, &rxd);
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index 6355deb..d20a4c3 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -38,7 +38,7 @@
 #define SPI_WRITE 0x20
 #define WRITE_DATA_MASK GENMASK_ULL(31, 0)
 
-#define SPI_MAX_RETRY 100000
+#define SPI_MAX_RETRY 1000000
 
 #define TYPE_SPI 0
 #define TYPE_NIOS_SPI 1
@@ -102,6 +102,7 @@ struct spi_transaction_dev {
 	struct altera_spi_device *dev;
 	int chipselect;
 	struct spi_tran_buffer *buffer;
+	pthread_mutex_t lock;
 };
 
 struct spi_tran_header {
diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 06ca625..013efee 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -154,6 +154,8 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	unsigned int rx_len = 0;
 	int retry = 0;
 	int spi_flags;
+	unsigned long timeout = msecs_to_timer_cycles(1000);
+	unsigned long ticks;
 	unsigned int resp_max_len = 2 * resp_len;
 
 	print_buffer("before bytes:", send_data, send_len);
@@ -207,8 +209,11 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	if (ret != SPI_FOUND_EOP) {
 		tx_buffer = NULL;
 		tx_len = 0;
-		if (retry++ > 10) {
-			dev_err(NULL, "cannot found valid data from SPI\n");
+		ticks = rte_get_timer_cycles();
+		if (time_after(ticks, timeout) &&
+				retry++ > SPI_MAX_RETRY) {
+			dev_err(NULL, "Have retry %d, found invalid packet data\n",
+				retry);
 			return -EBUSY;
 		}
 
@@ -427,23 +432,36 @@ static int do_transaction(struct spi_transaction_dev *dev, unsigned int addr,
 int spi_transaction_read(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_READ : SPI_TRAN_NON_SEQ_READ);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 int spi_transaction_write(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_WRITE : SPI_TRAN_NON_SEQ_WRITE);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 		int chipselect)
 {
 	struct spi_transaction_dev *spi_tran_dev;
+	int ret;
 
 	spi_tran_dev = opae_malloc(sizeof(struct spi_transaction_dev));
 	if (!spi_tran_dev)
@@ -453,18 +471,28 @@ struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 	spi_tran_dev->chipselect = chipselect;
 
 	spi_tran_dev->buffer = opae_malloc(sizeof(struct spi_tran_buffer));
-	if (!spi_tran_dev->buffer) {
-		opae_free(spi_tran_dev);
-		return NULL;
+	if (!spi_tran_dev->buffer)
+		goto err;
+
+	ret = pthread_mutex_init(&spi_tran_dev->lock, NULL);
+	if (ret) {
+		dev_err(spi_tran_dev, "fail to init mutex lock\n");
+		goto err;
 	}
 
 	return spi_tran_dev;
+
+err:
+	opae_free(spi_tran_dev);
+	return NULL;
 }
 
 void spi_transaction_remove(struct spi_transaction_dev *dev)
 {
 	if (dev && dev->buffer)
 		opae_free(dev->buffer);
-	if (dev)
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
 		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index cfcbf68..775731c 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -361,7 +361,7 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 	if (!mgr)
 		return -ENODEV;
 
-	opae_mgr_for_each_sensor(sensor) {
+	opae_mgr_for_each_sensor(mgr, sensor) {
 		if (!(sensor->flags & OPAE_SENSOR_VALID))
 			goto fail;
 
@@ -370,8 +370,8 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 			goto fail;
 
 		if (value == 0xdeadbeef) {
-			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
-					sensor->name, value);
+			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value %x\n",
+					raw_dev->dev_id, sensor->name, value);
 			continue;
 		}
 
@@ -394,8 +394,9 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 		/* monitor 12V AUX sensor */
 		if (!strcmp(sensor->name, "12V AUX Voltage")) {
 			if (value < AUX_VOLTAGE_WARN) {
-				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
-						sensor->name, value);
+				IFPGA_RAWDEV_PMD_INFO(
+					"%s reach theshold %d mV\n",
+					sensor->name, value);
 				*gsd_start = true;
 				break;
 			}
@@ -1433,6 +1434,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->bus = pci_dev->addr.bus;
+	data->devid = pci_dev->addr.devid;
+	data->function = pci_dev->addr.function;
 	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
-- 
1.8.3.1

[dpdk-dev] [PATCH v18 19/19] raw/ifpga: introducing new irq API

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing new register and unregister API for ifpga interrupt.
1. register FME and AFU interrupt
   ifpga_register_msix_irq()

2. unregister FME and AFU interrupt
   ifpga_unregister_msix_irq()

On PAC N3000 card, there is one PCIe MSIX interrupt for FME managerment,
like the error report, thermal management, we use this interrupt in
ifpga_rawdev device driver. on the other hand, there are about 4 PCIe
MSIX interrupts are reserved for AFU which end-user can use those
interrupts in their AFU logic design. End-user can use those APIs to
register interrupt handler in their AFU drivers.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 105 +++++++++++++++++++++++++++------------
 drivers/raw/ifpga/ifpga_rawdev.h |  14 ++++++
 2 files changed, 88 insertions(+), 31 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 775731c..e083fc5 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -79,6 +79,10 @@
 static int ifpga_monitor_start;
 static pthread_t ifpga_monitor_start_thread;
 
+#define IFPGA_MAX_IRQ 12
+/* 0 for FME interrupt, others are reserved for AFU irq */
+static struct rte_intr_handle ifpga_irq_handle[IFPGA_MAX_IRQ];
+
 static struct ifpga_rawdev *
 ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
 static int set_surprise_link_check_aer(
@@ -1328,53 +1332,90 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 	fme_err_handle_catfatal_error(mgr);
 }
 
-static struct rte_intr_handle fme_intr_handle;
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg)
+{
+	struct rte_intr_handle intr_handle;
+
+	if (type == IFPGA_FME_IRQ)
+		intr_handle = ifpga_irq_handle[0];
+	else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+	rte_intr_efd_disable(&intr_handle);
+
+	return rte_intr_callback_unregister(&intr_handle,
+			handler, arg);
+}
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg)
 {
 	int ret;
-	struct fpga_fme_err_irq_set err_irq_set;
+	struct rte_intr_handle intr_handle;
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_accelerator *acc;
 
-	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+	adapter = ifpga_rawdev_get_priv(dev);
+	if (!adapter)
+		return -ENODEV;
 
-	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
-	if (ret)
-		return -EINVAL;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
 
-	fme_intr_handle.fd = fme_intr_handle.efds[0];
+	if (type == IFPGA_FME_IRQ) {
+		intr_handle = ifpga_irq_handle[0];
+		count = 1;
+	} else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
-			fme_intr_handle.vfio_dev_fd,
-			fme_intr_handle.efds[0], fme_intr_handle.fd);
+	intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
 
-	err_irq_set.evtfd = fme_intr_handle.efds[0];
-	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	ret = rte_intr_efd_enable(&intr_handle, count);
 	if (ret)
-		return -EINVAL;
+		return -ENODEV;
+
+	intr_handle.fd = intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("register %s irq, vfio_fd=%d, fd=%d\n",
+			name, intr_handle.vfio_dev_fd,
+			intr_handle.fd);
+
+	if (type == IFPGA_FME_IRQ) {
+		struct fpga_fme_err_irq_set err_irq_set;
+		err_irq_set.evtfd = intr_handle.efds[0];
+
+		ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+		if (ret)
+			return -EINVAL;
+	} else if (type == IFPGA_AFU_IRQ) {
+		acc = opae_adapter_get_acc(adapter, port_id);
+		if (!acc)
+			return -EINVAL;
 
-	/* register FME interrupt using DPDK API */
-	ret = rte_intr_callback_register(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
+		ret = opae_acc_set_irq(acc, vec_start, count, intr_handle.efds);
+		if (ret)
+			return -EINVAL;
+	}
+
+	/* register interrupt handler using DPDK API */
+	ret = rte_intr_callback_register(&intr_handle,
+			handler, (void *)arg);
 	if (ret)
 		return -EINVAL;
 
-	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+	IFPGA_RAWDEV_PMD_INFO("success register %s interrupt\n", name);
 
 	return 0;
 }
 
 static int
-ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
-{
-	rte_intr_efd_disable(&fme_intr_handle);
-
-	return rte_intr_callback_unregister(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
-}
-
-static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -1463,7 +1504,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
-	ret = ifpga_register_fme_interrupt(mgr);
+	ret = ifpga_register_msix_irq(rawdev, 0, IFPGA_FME_IRQ, 0, 0,
+			fme_interrupt_handler, "fme_irq", mgr);
 	if (ret)
 		goto free_adapter_data;
 
@@ -1515,7 +1557,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	if (!mgr)
 		return -ENODEV;
 
-	if (ifpga_unregister_fme_interrupt(mgr))
+	if (ifpga_unregister_msix_irq(IFPGA_FME_IRQ, 0,
+				fme_interrupt_handler, mgr))
 		return -EINVAL;
 
 	opae_adapter_data_free(adapter->data);
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index bd42083..7754beb 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -62,4 +62,18 @@ struct ifpga_rawdev {
 struct ifpga_rawdev *
 ifpga_rawdev_get(const struct rte_rawdev *rawdev);
 
+enum ifpga_irq_type {
+	IFPGA_FME_IRQ = 0,
+	IFPGA_AFU_IRQ = 1,
+};
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg);
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 02/19] raw/ifpga/base: add irq support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME global error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

To build this feature, CONFIG_RTE_EAL_VFIO should be enabled.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                         |  2 +-
 config/common_linux                        |  6 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 59 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 19 ++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 18 +++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 19 ++++++++++
 6 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/config/common_base b/config/common_base
index 2323159..fef2f38 100644
--- a/config/common_base
+++ b/config/common_base
@@ -775,7 +775,7 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
 #
 # Compile PMD for Intel FPGA raw device
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
 
 #
 # Compile PMD for Intel IOAT raw device
diff --git a/config/common_linux b/config/common_linux
index 96e2e1f..a78b8c6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -68,3 +68,9 @@ CONFIG_RTE_LIBRTE_HINIC_PMD=y
 # Hisilicon HNS3 PMD driver
 #
 CONFIG_RTE_LIBRTE_HNS3_PMD=y
+
+#
+# Compile PMD for Intel FPGA raw device
+# To compile, CONFIG_RTE_EAL_VFIO should be enabled.
+#
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..0f852a7 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,61 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+#define FPGA_MAX_MSIX_VEC_COUNT	128
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MAX_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	/* 0 < irq_set->count < FPGA_MAX_MSIX_VEC_COUNT */
+	irq_set->count = count ?
+		(count > FPGA_MAX_MSIX_VEC_COUNT ?
+		 FPGA_MAX_MSIX_VEC_COUNT : count) : 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..2978c79 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,28 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&fme->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..c0aaf01 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,27 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..189f762 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,28 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 03/19] raw/ifpga/base: clear pending bit

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 2978c79..be041ec 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 04/19] raw/ifpga/base: add SEU error support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 31 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index be041ec..5d6d630 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,33 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr(struct ifpga_fme_hw *fme,
+		u64 *val, bool high)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+				FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	if (high)
+		*val = readq(&fme_err->seu_emr_h);
+	else
+		*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +297,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr(fme, &prop->data, 0);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr(fme, &prop->data, 1);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 05/19] raw/ifpga/base: add device tree support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

To eliminate build error, we currently do not compile raw/ifpga
and net/ipn3ke. User should install libfdt and libfdt-devel first,
modify config/common_linux, CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
to CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y, modify config/common_base,
CONFIG_RTE_LIBRTE_IPN3KE_PMD=n to CONFIG_RTE_LIBRTE_IPN3KE_PMD=y.
Then this function can work.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                        |   2 +-
 config/common_linux                       |   2 +-
 drivers/meson.build                       |   6 +-
 drivers/net/ipn3ke/meson.build            |  24 +++-
 drivers/raw/ifpga/base/meson.build        |   2 +-
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 drivers/raw/ifpga/meson.build             |  25 ++--
 mk/rte.app.mk                             |   2 +-
 9 files changed, 235 insertions(+), 21 deletions(-)

diff --git a/config/common_base b/config/common_base
index fef2f38..3c0ea7e 100644
--- a/config/common_base
+++ b/config/common_base
@@ -339,7 +339,7 @@ CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
 #
 # Compile burst-oriented IPN3KE PMD driver
 #
-CONFIG_RTE_LIBRTE_IPN3KE_PMD=y
+CONFIG_RTE_LIBRTE_IPN3KE_PMD=n
 
 #
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
diff --git a/config/common_linux b/config/common_linux
index a78b8c6..c5cf3d6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -73,4 +73,4 @@ CONFIG_RTE_LIBRTE_HNS3_PMD=y
 # Compile PMD for Intel FPGA raw device
 # To compile, CONFIG_RTE_EAL_VFIO should be enabled.
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
diff --git a/drivers/meson.build b/drivers/meson.build
index 4a1cb8b..7c0aed1 100644
--- a/drivers/meson.build
+++ b/drivers/meson.build
@@ -9,12 +9,12 @@ endif
 dpdk_driver_classes = ['common',
 	       'bus',
 	       'mempool', # depends on common and bus.
-	       'net',     # depends on common, bus and mempool.
+               'raw',     # depends on common and bus.
+	       'net',     # depends on common, bus, mempool and raw (raw provide API for net driver).
 	       'crypto',  # depends on common, bus and mempool (net in future).
 	       'compress', # depends on common, bus, mempool.
 	       'event',   # depends on common, bus, mempool and net.
-	       'baseband', # depends on common and bus.
-	       'raw']     # depends on common, bus, mempool, net and event.
+	       'baseband'] # depends on common and bus.
 
 disabled_drivers = get_option('disable_drivers').split(',')
 
diff --git a/drivers/net/ipn3ke/meson.build b/drivers/net/ipn3ke/meson.build
index 74b4d7c..e3c8a67 100644
--- a/drivers/net/ipn3ke/meson.build
+++ b/drivers/net/ipn3ke/meson.build
@@ -8,10 +8,22 @@
 #  rte_eth_dev_destroy()
 #  rte_eth_switch_domain_free()
 #
-allow_experimental_apis = true
 
-sources += files('ipn3ke_ethdev.c',
-	'ipn3ke_representor.c',
-	'ipn3ke_tm.c',
-	'ipn3ke_flow.c')
-deps += ['bus_ifpga', 'sched']
+dep = dependency('libfdt', required: false)
+if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
+	build = false
+	reason = 'missing dependency, "libfdt"'
+endif
+
+if build
+	allow_experimental_apis = true
+
+	sources += files('ipn3ke_ethdev.c',
+		'ipn3ke_representor.c',
+		'ipn3ke_tm.c',
+		'ipn3ke_flow.c')
+	deps += ['bus_ifpga', 'sched', 'pmd_i40e', 'rawdev', 'rawdev_ifpga']
+endif
diff --git a/drivers/raw/ifpga/base/meson.build b/drivers/raw/ifpga/base/meson.build
index 69f6598..b13e13e 100644
--- a/drivers/raw/ifpga/base/meson.build
+++ b/drivers/raw/ifpga/base/meson.build
@@ -25,5 +25,5 @@ sources = [
 
 base_lib = static_library('ifpga_rawdev_base', sources,
 	dependencies: static_rte_eal,
-	c_args: c_args)
+	c_args: cflags)
 base_objs = base_lib.extract_all_objects()
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
index 0ab6fd7..69debff 100644
--- a/drivers/raw/ifpga/meson.build
+++ b/drivers/raw/ifpga/meson.build
@@ -3,18 +3,27 @@
 
 version = 1
 
-subdir('base')
-objs = [base_objs]
-
 dep = dependency('libfdt', required: false)
 if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
 	build = false
 	reason = 'missing dependency, "libfdt"'
 endif
-deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
-	'bus_vdev', 'bus_ifpga', 'net']
-sources = files('ifpga_rawdev.c')
 
-includes += include_directories('base')
+if build
+	subdir('base')
+	objs = [base_objs]
+
+	deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
+		'bus_vdev', 'bus_ifpga', 'net']
+	ext_deps += dep
 
-allow_experimental_apis = true
+	sources = files('ifpga_rawdev.c')
+
+	includes += include_directories('base')
+	includes += include_directories('../../net/ipn3ke')
+
+	allow_experimental_apis = true
+endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index b91273f..28f3ef3 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -321,7 +321,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v13 05/19] raw/ifpga/base: add device tree support

[Bruce Richardson]

On Thu, Oct 24, 2019 at 07:38:25PM +0800, Andy Pei wrote:
> From: Tianfei zhang <tianfei.zhang@intel.com>
> 
> In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
> to manage the board configuration, like sensors, flash controller,
> QSFP, powers. And this is a SPI bus connected between A10 FPGA and
> MAX10, we can access the MAX10 registers over this SPI bus.
> 
> In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
> core temperature, Board temperature, board current, voltage and so on.
> 
> We use DTB (Device tree table) to describe it. This DTB file is store
> in nor flash partition, which will flashed in Factory when the boards
> delivery to customers. And the same time, the customers can easy to
> customizate the BMC configuration like change the sensors.
> 
> Add device tree support by using libfdt library in Linux distribution.
> The end-user should pre-install the libfdt and libfdt-devel package
> before use DPDK on PAC N3000 Card.
> 
> For Centos 7.x: sudo yum install libfdt libfdt-devel
> For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1
> 
> To eliminate build error, we currently do not compile raw/ifpga
> and net/ipn3ke. User should install libfdt and libfdt-devel first,
> modify config/common_linux, CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
> to CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y, modify config/common_base,
> CONFIG_RTE_LIBRTE_IPN3KE_PMD=n to CONFIG_RTE_LIBRTE_IPN3KE_PMD=y.
> Then this function can work.
> 
> Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  config/common_base                        |   2 +-
>  config/common_linux                       |   2 +-
>  drivers/meson.build                       |   6 +-
>  drivers/net/ipn3ke/meson.build            |  24 +++-
>  drivers/raw/ifpga/base/meson.build        |   2 +-
>  drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
>  drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
>  drivers/raw/ifpga/meson.build             |  25 ++--
>  mk/rte.app.mk                             |   2 +-
>  9 files changed, 235 insertions(+), 21 deletions(-)
> 
> diff --git a/config/common_base b/config/common_base
> index fef2f38..3c0ea7e 100644
> --- a/config/common_base
> +++ b/config/common_base
> @@ -339,7 +339,7 @@ CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
>  #
>  # Compile burst-oriented IPN3KE PMD driver
>  #
> -CONFIG_RTE_LIBRTE_IPN3KE_PMD=y
> +CONFIG_RTE_LIBRTE_IPN3KE_PMD=n
>  
>  #
>  # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
> diff --git a/config/common_linux b/config/common_linux
> index a78b8c6..c5cf3d6 100644
> --- a/config/common_linux
> +++ b/config/common_linux
> @@ -73,4 +73,4 @@ CONFIG_RTE_LIBRTE_HNS3_PMD=y
>  # Compile PMD for Intel FPGA raw device
>  # To compile, CONFIG_RTE_EAL_VFIO should be enabled.
>  #
> -CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
> +CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
> diff --git a/drivers/meson.build b/drivers/meson.build
> index 4a1cb8b..7c0aed1 100644
> --- a/drivers/meson.build
> +++ b/drivers/meson.build
> @@ -9,12 +9,12 @@ endif
>  dpdk_driver_classes = ['common',
>  	       'bus',
>  	       'mempool', # depends on common and bus.
> -	       'net',     # depends on common, bus and mempool.
> +               'raw',     # depends on common and bus.
> +	       'net',     # depends on common, bus, mempool and raw (raw provide API for net driver).

The comment in brackets is unnecessary here, since that is what is implied
by saying that there is a dependency on net. Thinking about it more, I
think it would be useful in future to make this more concrete for
debugging, and specify what drivers in one set depend on elements in
another [assuming it's only one or two]. It's not as though all net drivers
depend on the rawdevs.

>  	       'crypto',  # depends on common, bus and mempool (net in future).
>  	       'compress', # depends on common, bus, mempool.
>  	       'event',   # depends on common, bus, mempool and net.
> -	       'baseband', # depends on common and bus.
> -	       'raw']     # depends on common, bus, mempool, net and event.

I assume that the comment here is incorrect, and that no rawdev does depend
on event drivers or net drivers? Have you checked each driver for such a
dependency?

> +	       'baseband'] # depends on common and bus.
>  
>  disabled_drivers = get_option('disable_drivers').split(',')
>

Assuming that the comment is incorrect, and that no existing rawdev driver
depends on a net or event driver, no issues with this meson change.

Acked-by: Bruce Richardson <bruce.richardson@intel.com>

Re: [dpdk-dev] [PATCH v13 05/19] raw/ifpga/base: add device tree support

[Xu, Rosen]

> -----Original Message-----
> From: Bruce Richardson [mailto:bruce.richardson@intel.com]
> Sent: Thursday, October 24, 2019 21:09
> To: Pei, Andy <andy.pei@intel.com>
> Cc: dev@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> <tianfei.zhang@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>; Zhang, Qi
> Z <qi.z.zhang@intel.com>; Yigit, Ferruh <ferruh.yigit@intel.com>
> Subject: Re: [PATCH v13 05/19] raw/ifpga/base: add device tree support
> 
> On Thu, Oct 24, 2019 at 07:38:25PM +0800, Andy Pei wrote:
> > From: Tianfei zhang <tianfei.zhang@intel.com>
> >
> > In PAC N3000 card, this is a BMC chip which using MAX10 FPGA to manage
> > the board configuration, like sensors, flash controller, QSFP, powers.
> > And this is a SPI bus connected between A10 FPGA and MAX10, we can
> > access the MAX10 registers over this SPI bus.
> >
> > In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
> > core temperature, Board temperature, board current, voltage and so on.
> >
> > We use DTB (Device tree table) to describe it. This DTB file is store
> > in nor flash partition, which will flashed in Factory when the boards
> > delivery to customers. And the same time, the customers can easy to
> > customizate the BMC configuration like change the sensors.
> >
> > Add device tree support by using libfdt library in Linux distribution.
> > The end-user should pre-install the libfdt and libfdt-devel package
> > before use DPDK on PAC N3000 Card.
> >
> > For Centos 7.x: sudo yum install libfdt libfdt-devel For Ubuntu 18.04:
> > sudo apt install libfdt-dev libfdt1
> >
> > To eliminate build error, we currently do not compile raw/ifpga and
> > net/ipn3ke. User should install libfdt and libfdt-devel first, modify
> > config/common_linux, CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
> > to CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y, modify
> config/common_base,
> > CONFIG_RTE_LIBRTE_IPN3KE_PMD=n to
> CONFIG_RTE_LIBRTE_IPN3KE_PMD=y.
> > Then this function can work.
> >
> > Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
> > Signed-off-by: Andy Pei <andy.pei@intel.com>
> > ---
> >  config/common_base                        |   2 +-
> >  config/common_linux                       |   2 +-
> >  drivers/meson.build                       |   6 +-
> >  drivers/net/ipn3ke/meson.build            |  24 +++-
> >  drivers/raw/ifpga/base/meson.build        |   2 +-
> >  drivers/raw/ifpga/base/opae_intel_max10.c | 183
> > ++++++++++++++++++++++++++++++
> drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
> >  drivers/raw/ifpga/meson.build             |  25 ++--
> >  mk/rte.app.mk                             |   2 +-
> >  9 files changed, 235 insertions(+), 21 deletions(-)
> >
> > diff --git a/config/common_base b/config/common_base index
> > fef2f38..3c0ea7e 100644
> > --- a/config/common_base
> > +++ b/config/common_base
> > @@ -339,7 +339,7 @@ CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
> >  #
> >  # Compile burst-oriented IPN3KE PMD driver  #
> > -CONFIG_RTE_LIBRTE_IPN3KE_PMD=y
> > +CONFIG_RTE_LIBRTE_IPN3KE_PMD=n
> >
> >  #
> >  # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD diff --git
> > a/config/common_linux b/config/common_linux index a78b8c6..c5cf3d6
> > 100644
> > --- a/config/common_linux
> > +++ b/config/common_linux
> > @@ -73,4 +73,4 @@ CONFIG_RTE_LIBRTE_HNS3_PMD=y  # Compile PMD
> for
> > Intel FPGA raw device  # To compile, CONFIG_RTE_EAL_VFIO should be
> > enabled.
> >  #
> > -CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
> > +CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
> > diff --git a/drivers/meson.build b/drivers/meson.build index
> > 4a1cb8b..7c0aed1 100644
> > --- a/drivers/meson.build
> > +++ b/drivers/meson.build
> > @@ -9,12 +9,12 @@ endif
> >  dpdk_driver_classes = ['common',
> >  	       'bus',
> >  	       'mempool', # depends on common and bus.
> > -	       'net',     # depends on common, bus and mempool.
> > +               'raw',     # depends on common and bus.
> > +	       'net',     # depends on common, bus, mempool and raw (raw
> provide API for net driver).
> 
> The comment in brackets is unnecessary here, since that is what is implied by
> saying that there is a dependency on net. Thinking about it more, I think it
> would be useful in future to make this more concrete for debugging, and
> specify what drivers in one set depend on elements in another [assuming it's
> only one or two]. It's not as though all net drivers depend on the rawdevs.

Okay, will remove it.

> >  	       'crypto',  # depends on common, bus and mempool (net in
> future).
> >  	       'compress', # depends on common, bus, mempool.
> >  	       'event',   # depends on common, bus, mempool and net.
> > -	       'baseband', # depends on common and bus.
> > -	       'raw']     # depends on common, bus, mempool, net and event.
> 
> I assume that the comment here is incorrect, and that no rawdev does
> depend on event drivers or net drivers? Have you checked each driver for
> such a dependency?

Yes, we have checked it. And build it.

> > +	       'baseband'] # depends on common and bus.
> >
> >  disabled_drivers = get_option('disable_drivers').split(',')
> >
> 
> Assuming that the comment is incorrect, and that no existing rawdev driver
> depends on a net or event driver, no issues with this meson change.
> 
> Acked-by: Bruce Richardson <bruce.richardson@intel.com>

Thanks a lot.

Re: [dpdk-dev] [PATCH v13 05/19] raw/ifpga/base: add device tree support

[Ye Xiaolong]

On 10/24, Andy Pei wrote:
>From: Tianfei zhang <tianfei.zhang@intel.com>
>--- a/drivers/meson.build
>+++ b/drivers/meson.build
>@@ -9,12 +9,12 @@ endif
> dpdk_driver_classes = ['common',
> 	       'bus',
> 	       'mempool', # depends on common and bus.
>-	       'net',     # depends on common, bus and mempool.
>+               'raw',     # depends on common and bus.
>+	       'net',     # depends on common, bus, mempool and raw (raw provide API for net driver).

Minor nit, please keep 'raw' align with other lines.

Thanks,
Xiaolong

> 	       'crypto',  # depends on common, bus and mempool (net in future).
> 	       'compress', # depends on common, bus, mempool.
> 	       'event',   # depends on common, bus, mempool and net.
>-	       'baseband', # depends on common and bus.
>-	       'raw']     # depends on common, bus, mempool, net and event.
>+	       'baseband'] # depends on common and bus.
> 
> disabled_drivers = get_option('disable_drivers').split(',')
> 

[dpdk-dev] [PATCH v13 06/19] raw/ifpga/base: align the send buffer for SPI

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 07/19] raw/ifpga/base: add sensor support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 08/19] raw/ifpga/base: introducing sensor APIs

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 09/19] raw/ifpga/base: update SEU register definition

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 10/19] raw/ifpga: add SEU error handler

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 245 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 1825143..f5e6119 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -27,6 +27,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -605,6 +607,236 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -652,6 +884,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -677,6 +910,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -696,6 +933,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -720,6 +958,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 11/19] raw/ifpga: add PCIe BDF devices tree scan

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 551 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 562 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index f5e6119..01ff76a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_malloc.h>
@@ -17,7 +19,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -25,6 +27,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -37,6 +40,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -64,6 +73,494 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(
+		struct ifpga_rawdev *ifpga_rdev, int force_disable);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+		int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
+					sensor->name, value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(
+	struct ifpga_rawdev *ifpga_rdev, int force_disable)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable || force_disable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev,
+					gsd_enable);
+				if (ret == 1 && !gsd_enable) {
+					gsd_enable = 1;
+					i = -1;
+				}
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -372,8 +869,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -842,6 +1340,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -855,7 +1354,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -869,6 +1368,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -987,6 +1494,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1018,13 +1526,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1038,7 +1565,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1065,6 +1593,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 12/19] net/ipn3ke: remove configuration for i40e port bonding

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/ipn3ke/Makefile                    |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c             | 289 +++----------------------
 drivers/net/ipn3ke/ipn3ke_representor.c        |   8 +-
 drivers/raw/ifpga/rte_rawdev_ifpga_version.map |   3 +
 4 files changed, 47 insertions(+), 255 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..2c65e49 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index 28d8aaf..3051cdf 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -324,7 +325,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -342,6 +344,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -409,9 +417,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -462,7 +467,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -489,7 +498,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -501,6 +515,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -508,6 +538,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -553,254 +584,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index d37f5e2..7e5d29d 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2918,8 +2919,11 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		rte_pmd_i40e_set_switch_dev(rpst->i40e_pf_eth_port_id,
+					    rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
diff --git a/drivers/raw/ifpga/rte_rawdev_ifpga_version.map b/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
index 9b9ab1a..a6b50aa 100644
--- a/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
+++ b/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
@@ -1,4 +1,7 @@
 DPDK_18.05 {
+	global:
+
+	ifpga_rawdev_get;
 
 	local: *;
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 13/19] raw/ifpga/base: add secure support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 137 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 ++++++++++++-----
 4 files changed, 198 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..748ab56 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,63 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager")) {
+			sysmgr_offset = offset;
+			break;
+		}
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +576,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 14/19] raw/ifpga/base: configure FEC mode

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 15/19] raw/ifpga/base: clean fme errors

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occurred.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 5d6d630..5905eac 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 01ff76a..95f079a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1174,6 +1174,25 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1183,6 +1202,9 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 16/19] raw/ifpga/base: add new API get board info

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..9f0147d 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disaggregate;
+	u8 lightweight;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..799d67d 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweight = id.lightweiht;
+	fme->board_info.disaggregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweight:%s\n"
+			"disaggregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweight),
+			check_support(fme->board_info.disaggregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 17/19] raw/ifpga: add lightweight fpga image support

[Andy Pei]

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 95f079a..e87af66 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -835,6 +835,8 @@ static int set_surprise_link_check_aer(
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -861,22 +863,40 @@ static int set_surprise_link_check_aer(
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweight) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
-		uuid.b + 8, sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 18/19] raw/ifpga/base: add multiple cards support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, there is one MAX10 chip in each card, and
all of the sensors are connected to MAX10 chip. To support multiple
cards in one server, we introducing a sensor device list under
intel_max10_device instead of a global list. On the other hand, we
using separate intel_max10_device instance for each opae_adatper.

Add mutex lock on do_transaction() function for SPI driver to avoid
race condition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c            |  40 +++++++---
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          |  14 ++--
 drivers/raw/ifpga/base/opae_hw_api.h          |  15 ++--
 drivers/raw/ifpga/base/opae_i2c.c             |  44 ++++++++---
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_intel_max10.c     | 110 ++++++++++++++------------
 drivers/raw/ifpga/base/opae_intel_max10.h     |  19 +++--
 drivers/raw/ifpga/base/opae_spi.c             |   5 --
 drivers/raw/ifpga/base/opae_spi.h             |   3 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  44 +++++++++--
 drivers/raw/ifpga/ifpga_rawdev.c              |  14 ++--
 12 files changed, 205 insertions(+), 109 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 799d67d..c31a94c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -839,8 +839,13 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	struct ifpga_hw *hw;
 	u32 val;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
@@ -854,7 +859,10 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	fme->board_info.seu = id.seu;
 	fme->board_info.ptp = id.ptp;
 
-	dev_info(fme, "found: board: %s type: %s\n",
+	dev_info(fme, "found: PCI dev: %02x:%02x:%x board: %s type: %s\n",
+			hw->pci_data->bus,
+			hw->pci_data->devid,
+			hw->pci_data->function,
 			board_major_to_string(fme->board_info.major),
 			board_type_to_string(fme->board_info.type));
 
@@ -882,11 +890,11 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
-	if (max10_sys_read(MAX10_BUILD_VER, &val))
+	if (max10_sys_read(fme->max10_dev, MAX10_BUILD_VER, &val))
 		return -EINVAL;
 	fme->board_info.max10_version = val & 0xffffff;
 
-	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+	if (max10_sys_read(fme->max10_dev, NIOS2_FW_VERSION, &val))
 		return -EINVAL;
 	fme->board_info.nios_fw_version = val & 0xffffff;
 
@@ -897,12 +905,12 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	return 0;
 }
 
-static int spi_self_checking(void)
+static int spi_self_checking(struct intel_max10_device *dev)
 {
 	u32 val;
 	int ret;
 
-	ret = max10_sys_read(MAX10_TEST_REG, &val);
+	ret = max10_sys_read(dev, MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
@@ -937,10 +945,11 @@ static int fme_spi_init(struct ifpga_feature *feature)
 		goto spi_fail;
 	}
 
+
 	fme->max10_dev = max10;
 
 	/* SPI self test */
-	if (spi_self_checking()) {
+	if (spi_self_checking(max10)) {
 		ret = -EIO;
 		goto max10_fail;
 	}
@@ -1041,8 +1050,18 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 	struct ifpga_fme_hw *fme = (struct ifpga_fme_hw *)feature->parent;
 	struct altera_spi_device *spi_master;
 	struct intel_max10_device *max10;
+	struct ifpga_hw *hw;
+	struct opae_manager *mgr;
 	int ret = 0;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
+	mgr = hw->adapter->mgr;
+	if (!mgr)
+		return -ENODEV;
+
 	dev_info(fme, "FME SPI Master (NIOS) Init.\n");
 	dev_debug(fme, "FME SPI base addr %p.\n",
 			feature->addr);
@@ -1080,12 +1099,15 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 		goto release_dev;
 	}
 
+	max10->bus = hw->pci_data->bus;
+
 	fme_get_board_interface(fme);
 
 	fme->max10_dev = max10;
+	mgr->sensor_list = &max10->opae_sensor_list;
 
 	/* SPI self test */
-	if (spi_self_checking())
+	if (spi_self_checking(max10))
 		goto spi_fail;
 
 	return ret;
@@ -1344,7 +1366,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(dev, PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1372,7 +1394,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(sensor->value_reg, value)) {
+	if (max10_sys_read(dev, sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_debug.c b/drivers/raw/ifpga/base/opae_debug.c
index 88f2d5c..dad3ea3 100644
--- a/drivers/raw/ifpga/base/opae_debug.c
+++ b/drivers/raw/ifpga/base/opae_debug.c
@@ -59,6 +59,9 @@ static void opae_adapter_data_dump(void *data)
 		opae_log("OPAE Adapter Type = PCI\n");
 		opae_log("PCI Device ID: 0x%04x\n", d_pci->device_id);
 		opae_log("PCI Vendor ID: 0x%04x\n", d_pci->vendor_id);
+		opae_log("PCI bus: 0x%04x\n", d_pci->bus);
+		opae_log("PCI devid: 0x%04x\n", d_pci->devid);
+		opae_log("PCI function: 0x%04x\n", d_pci->function);
 
 		for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 			r = &d_pci->region[i];
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 1ccc967..c969dfe 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -583,11 +583,12 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_id(unsigned int id)
+opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (sensor->id == id)
 			return sensor;
 
@@ -601,11 +602,12 @@ struct opae_sensor_info *
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_name(const char *name)
+opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (!strcmp(sensor->name, name))
 			return sensor;
 
@@ -630,7 +632,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_name(name);
+	sensor = opae_mgr_get_sensor_by_name(mgr, name);
 	if (!sensor)
 		return -ENODEV;
 
@@ -658,7 +660,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_id(id);
+	sensor = opae_mgr_get_sensor_by_id(mgr, id);
 	if (!sensor)
 		return -ENODEV;
 
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index b78fbd5..cdf369f 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -40,6 +40,7 @@ struct opae_manager {
 	struct opae_adapter *adapter;
 	struct opae_manager_ops *ops;
 	struct opae_manager_networking_ops *network_ops;
+	struct opae_sensor_list *sensor_list;
 	void *data;
 };
 
@@ -75,9 +76,8 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
-extern struct opae_sensor_list opae_sensor_list;
-#define opae_mgr_for_each_sensor(sensor) \
-	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+#define opae_mgr_for_each_sensor(mgr, sensor) \
+	TAILQ_FOREACH(sensor, mgr->sensor_list, node)
 
 /* OPAE Manager APIs */
 struct opae_manager *
@@ -88,8 +88,10 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
-struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
-struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id);
 int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
 		const char *name, unsigned int *value);
 int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
@@ -241,6 +243,9 @@ struct opae_adapter_data_pci {
 	enum opae_adapter_type type;
 	u16 device_id;
 	u16 vendor_id;
+	u16 bus; /*Device bus for PCI */
+	u16 devid; /* Device ID */
+	u16 function; /* Device function */
 	struct opae_reg_region region[PCI_MAX_RESOURCE];
 	int vfio_dev_fd;  /* VFIO device file descriptor */
 };
diff --git a/drivers/raw/ifpga/base/opae_i2c.c b/drivers/raw/ifpga/base/opae_i2c.c
index f8bc247..846d751 100644
--- a/drivers/raw/ifpga/base/opae_i2c.c
+++ b/drivers/raw/ifpga/base/opae_i2c.c
@@ -28,6 +28,9 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 {
 	u8 msgbuf[2];
 	int i = 0;
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
 
 	if (flags & I2C_FLAG_ADDR16)
 		msgbuf[i++] = offset >> 8;
@@ -49,10 +52,16 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		},
 	};
 
-	if (!dev->xfer)
-		return -ENODEV;
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	ret = i2c_transfer(dev, msg, 2);
 
-	return i2c_transfer(dev, msg, 2);
+exit:
+	pthread_mutex_unlock(&dev->lock);
+	return ret;
 }
 
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
@@ -63,12 +72,18 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	int ret;
 	int i = 0;
 
-	if (!dev->xfer)
-		return -ENODEV;
+	pthread_mutex_lock(&dev->lock);
+
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
 
 	buf = opae_malloc(I2C_MAX_OFFSET_LEN + len);
-	if (!buf)
-		return -ENOMEM;
+	if (!buf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
 
 	msg.addr = slave_addr;
 	msg.flags = 0;
@@ -84,6 +99,8 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	ret = i2c_transfer(dev, &msg, 1);
 
 	opae_free(buf);
+exit:
+	pthread_mutex_unlock(&dev->lock);
 	return ret;
 }
 
@@ -477,14 +494,19 @@ struct altera_i2c_dev *altera_i2c_probe(void *base)
 	dev->i2c_clk = dev->i2c_param.ref_clk * 1000000;
 	dev->xfer = altera_i2c_xfer;
 
+	if (pthread_mutex_init(&dev->lock, NULL))
+		return NULL;
+
 	altera_i2c_hardware_init(dev);
 
 	return dev;
 }
 
-int altera_i2c_remove(struct altera_i2c_dev *dev)
+void altera_i2c_remove(struct altera_i2c_dev *dev)
 {
-	altera_i2c_disable(dev);
-
-	return 0;
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
+		altera_i2c_disable(dev);
+		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/base/opae_i2c.h b/drivers/raw/ifpga/base/opae_i2c.h
index 8890c8f..266e127 100644
--- a/drivers/raw/ifpga/base/opae_i2c.h
+++ b/drivers/raw/ifpga/base/opae_i2c.h
@@ -93,6 +93,7 @@ struct altera_i2c_dev {
 	u32 isr_mask;
 	u8 *buf;
 	int (*xfer)(struct altera_i2c_dev *dev, struct i2c_msg *msg, int num);
+	pthread_mutex_t lock;
 };
 
 /**
@@ -114,7 +115,7 @@ enum i2c_msg_flags {
 };
 
 struct altera_i2c_dev *altera_i2c_probe(void *base);
-int altera_i2c_remove(struct altera_i2c_dev *dev);
+void altera_i2c_remove(struct altera_i2c_dev *dev);
 int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		u32 offset, u8 *buf, u32 count);
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 748ab56..8e23ca1 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -5,45 +5,50 @@
 #include "opae_intel_max10.h"
 #include <libfdt.h>
 
-static struct intel_max10_device *g_max10;
-
-struct opae_sensor_list opae_sensor_list =
-	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
-
-int max10_reg_read(unsigned int reg, unsigned int *val)
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_read(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x\n", __func__, dev->bus, reg);
+
+	return spi_transaction_read(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)val);
 }
 
-int max10_reg_write(unsigned int reg, unsigned int val)
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val)
 {
 	unsigned int tmp = val;
 
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_write(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x, val:0x%x\n", __func__,
+			dev->bus, reg, val);
+
+	return spi_transaction_write(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)&tmp);
 }
 
-int max10_sys_read(unsigned int offset, unsigned int *val)
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_read(g_max10->base + offset, val);
+
+	return max10_reg_read(dev, dev->base + offset, val);
 }
 
-int max10_sys_write(unsigned int offset, unsigned int val)
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_write(g_max10->base + offset, val);
+	return max10_reg_write(dev, dev->base + offset, val);
 }
 
 static struct max10_compatible_id max10_id_table[] = {
@@ -86,8 +91,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
-static int altera_nor_flash_read(u32 offset,
-		void *buffer, u32 len)
+static int altera_nor_flash_read(struct intel_max10_device *dev,
+	u32 offset, void *buffer, u32 len)
 {
 	int word_len;
 	int i;
@@ -95,13 +100,13 @@ static int altera_nor_flash_read(u32 offset,
 	unsigned int value;
 	int ret;
 
-	if (!buffer || len <= 0)
+	if (!dev || !buffer || len <= 0)
 		return -ENODEV;
 
 	word_len = len/4;
 
 	for (i = 0; i < word_len; i++) {
-		ret = max10_reg_read(offset + i*4,
+		ret = max10_reg_read(dev, offset + i*4,
 				&value);
 		if (ret)
 			return -EBUSY;
@@ -112,12 +117,12 @@ static int altera_nor_flash_read(u32 offset,
 	return 0;
 }
 
-static int enable_nor_flash(bool on)
+static int enable_nor_flash(struct intel_max10_device *dev, bool on)
 {
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_sys_read(RSU_REG, &val);
+	ret = max10_sys_read(dev, RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -128,7 +133,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_sys_write(RSU_REG, val);
+	return max10_sys_write(dev, RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -140,7 +145,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_sys_read(DT_AVAIL_REG, &val);
+	ret = max10_sys_read(max10, DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -151,19 +156,19 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
+	ret = max10_sys_read(max10, DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
 	}
 
-	ret = enable_nor_flash(true);
+	ret = enable_nor_flash(max10, true);
 	if (ret) {
 		dev_err(max10 "fail to enable flash\n");
 		return ret;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	ret = altera_nor_flash_read(max10, dt_addr, &hdr, sizeof(hdr));
 	if (ret) {
 		dev_err(max10 "read fdt header fail\n");
 		goto done;
@@ -188,7 +193,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		goto done;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	ret = altera_nor_flash_read(max10, dt_addr, fdt_root, dt_size);
 	if (ret) {
 		dev_err(max10 "cannot read device table\n");
 		goto done;
@@ -207,7 +212,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	max10->fdt_root = fdt_root;
 
 done:
-	ret = enable_nor_flash(false);
+	ret = enable_nor_flash(max10, false);
 
 	if (ret && fdt_root)
 		opae_free(fdt_root);
@@ -298,12 +303,12 @@ static int fdt_get_named_reg(const void *fdt, int node, const char *name,
 	return fdt_get_reg(fdt, node, idx, start, size);
 }
 
-static void max10_sensor_uinit(void)
+static void max10_sensor_uinit(struct intel_max10_device *dev)
 {
 	struct opae_sensor_info *info;
 
-	TAILQ_FOREACH(info, &opae_sensor_list, node) {
-		TAILQ_REMOVE(&opae_sensor_list, info, node);
+	TAILQ_FOREACH(info, &dev->opae_sensor_list, node) {
+		TAILQ_REMOVE(&dev->opae_sensor_list, info, node);
 		opae_free(info);
 	}
 }
@@ -313,8 +318,8 @@ static bool sensor_reg_valid(struct sensor_reg *reg)
 	return !!reg->size;
 }
 
-static int max10_add_sensor(struct raw_sensor_info *info,
-		struct opae_sensor_info *sensor)
+static int max10_add_sensor(struct intel_max10_device *dev,
+	struct raw_sensor_info *info, struct opae_sensor_info *sensor)
 {
 	int i;
 	int ret = 0;
@@ -332,12 +337,15 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_sys_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(dev, info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
-		if (val == 0xdeadbeef)
+		if (val == 0xdeadbeef) {
+			dev_debug(dev, "%s: sensor:%s invalid 0x%x at:%d\n",
+				__func__, sensor->name, val, i);
 			continue;
+		}
 
 		val *= info->multiplier;
 
@@ -458,7 +466,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
 		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
 
-		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+		dev_debug(dev, "found sensor from DTB: %s: %s: %u: %u\n",
 				raw->name, raw->type,
 				raw->id, raw->multiplier);
 
@@ -473,15 +481,16 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 			goto free_sensor;
 		}
 
-		if (max10_add_sensor(raw, sensor)) {
+		if (max10_add_sensor(dev, raw, sensor)) {
 			ret = -EINVAL;
 			opae_free(sensor);
 			goto free_sensor;
 		}
 
-		if (sensor->flags & OPAE_SENSOR_VALID)
-			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
-		else
+		if (sensor->flags & OPAE_SENSOR_VALID) {
+			TAILQ_INSERT_TAIL(&dev->opae_sensor_list, sensor, node);
+			dev_info(dev, "found valid sensor: %s\n", sensor->name);
+		} else
 			opae_free(sensor);
 
 		opae_free(raw);
@@ -492,7 +501,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 free_sensor:
 	if (raw)
 		opae_free(raw);
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 	return ret;
 }
 
@@ -500,7 +509,7 @@ static int check_max10_version(struct intel_max10_device *dev)
 {
 	unsigned int v;
 
-	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+	if (!max10_reg_read(dev, MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
 				&v)) {
 		if (v != 0xffffffff) {
 			dev_info(dev, "secure MAX10 detected\n");
@@ -565,6 +574,8 @@ struct intel_max10_device *
 	if (!dev)
 		return NULL;
 
+	TAILQ_INIT(&dev->opae_sensor_list);
+
 	dev->spi_master = spi;
 
 	dev->spi_tran_dev = spi_transaction_init(spi, chipselect);
@@ -573,9 +584,6 @@ struct intel_max10_device *
 		goto free_dev;
 	}
 
-	/* set the max10 device firstly */
-	g_max10 = dev;
-
 	/* check the max10 version */
 	ret = check_max10_version(dev);
 	if (ret) {
@@ -601,7 +609,7 @@ struct intel_max10_device *
 	}
 
 	/* read FPGA loading information */
-	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
+	ret = max10_sys_read(dev, FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
 		goto release_max10_hw;
@@ -611,14 +619,13 @@ struct intel_max10_device *
 	return dev;
 
 release_max10_hw:
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return NULL;
@@ -629,7 +636,7 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
@@ -637,7 +644,6 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index e632941..123cdc4 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -26,6 +26,9 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SECURE		BIT(6)
 #define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
@@ -33,6 +36,8 @@ struct intel_max10_device {
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
 	unsigned int base; /* max10 base address */
+	u16 bus;
+	struct opae_sensor_list opae_sensor_list;
 };
 
 /* retimer speed */
@@ -136,17 +141,19 @@ struct opae_retimer_status {
 
 #define DFT_MAX_SIZE		0x7e0000
 
-int max10_reg_read(unsigned int reg, unsigned int *val);
-int max10_reg_write(unsigned int reg, unsigned int val);
-int max10_sys_read(unsigned int offset, unsigned int *val);
-int max10_sys_write(unsigned int offset, unsigned int val);
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val);
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val);
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val);
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
-/** List of opae sensors */
-TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
 
 #define SENSOR_REG_VALUE 0x0
 #define SENSOR_REG_HIGH_WARN 0x1
diff --git a/drivers/raw/ifpga/base/opae_spi.c b/drivers/raw/ifpga/base/opae_spi.c
index cc52782..bfdc83e 100644
--- a/drivers/raw/ifpga/base/opae_spi.c
+++ b/drivers/raw/ifpga/base/opae_spi.c
@@ -181,7 +181,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 	u32 rxd;
 	unsigned int tx_data;
 	u32 status;
-	int retry = 0;
 	int ret;
 
 	while (count < dev->len) {
@@ -194,10 +193,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 				return -EIO;
 			if (status & ALTERA_SPI_STATUS_RRDY_MSK)
 				break;
-			if (retry++ > SPI_MAX_RETRY) {
-				dev_err(dev, "%s, read timeout\n", __func__);
-				return -EBUSY;
-			}
 		}
 
 		ret = spi_reg_read(dev, ALTERA_SPI_RXDATA, &rxd);
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index 6355deb..d20a4c3 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -38,7 +38,7 @@
 #define SPI_WRITE 0x20
 #define WRITE_DATA_MASK GENMASK_ULL(31, 0)
 
-#define SPI_MAX_RETRY 100000
+#define SPI_MAX_RETRY 1000000
 
 #define TYPE_SPI 0
 #define TYPE_NIOS_SPI 1
@@ -102,6 +102,7 @@ struct spi_transaction_dev {
 	struct altera_spi_device *dev;
 	int chipselect;
 	struct spi_tran_buffer *buffer;
+	pthread_mutex_t lock;
 };
 
 struct spi_tran_header {
diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 06ca625..013efee 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -154,6 +154,8 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	unsigned int rx_len = 0;
 	int retry = 0;
 	int spi_flags;
+	unsigned long timeout = msecs_to_timer_cycles(1000);
+	unsigned long ticks;
 	unsigned int resp_max_len = 2 * resp_len;
 
 	print_buffer("before bytes:", send_data, send_len);
@@ -207,8 +209,11 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	if (ret != SPI_FOUND_EOP) {
 		tx_buffer = NULL;
 		tx_len = 0;
-		if (retry++ > 10) {
-			dev_err(NULL, "cannot found valid data from SPI\n");
+		ticks = rte_get_timer_cycles();
+		if (time_after(ticks, timeout) &&
+				retry++ > SPI_MAX_RETRY) {
+			dev_err(NULL, "Have retry %d, found invalid packet data\n",
+				retry);
 			return -EBUSY;
 		}
 
@@ -427,23 +432,36 @@ static int do_transaction(struct spi_transaction_dev *dev, unsigned int addr,
 int spi_transaction_read(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_READ : SPI_TRAN_NON_SEQ_READ);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 int spi_transaction_write(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_WRITE : SPI_TRAN_NON_SEQ_WRITE);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 		int chipselect)
 {
 	struct spi_transaction_dev *spi_tran_dev;
+	int ret;
 
 	spi_tran_dev = opae_malloc(sizeof(struct spi_transaction_dev));
 	if (!spi_tran_dev)
@@ -453,18 +471,28 @@ struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 	spi_tran_dev->chipselect = chipselect;
 
 	spi_tran_dev->buffer = opae_malloc(sizeof(struct spi_tran_buffer));
-	if (!spi_tran_dev->buffer) {
-		opae_free(spi_tran_dev);
-		return NULL;
+	if (!spi_tran_dev->buffer)
+		goto err;
+
+	ret = pthread_mutex_init(&spi_tran_dev->lock, NULL);
+	if (ret) {
+		dev_err(spi_tran_dev, "fail to init mutex lock\n");
+		goto err;
 	}
 
 	return spi_tran_dev;
+
+err:
+	opae_free(spi_tran_dev);
+	return NULL;
 }
 
 void spi_transaction_remove(struct spi_transaction_dev *dev)
 {
 	if (dev && dev->buffer)
 		opae_free(dev->buffer);
-	if (dev)
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
 		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index e87af66..7253644 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -360,7 +360,7 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 	if (!mgr)
 		return -ENODEV;
 
-	opae_mgr_for_each_sensor(sensor) {
+	opae_mgr_for_each_sensor(mgr, sensor) {
 		if (!(sensor->flags & OPAE_SENSOR_VALID))
 			goto fail;
 
@@ -369,8 +369,8 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 			goto fail;
 
 		if (value == 0xdeadbeef) {
-			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
-					sensor->name, value);
+			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value %x\n",
+					raw_dev->dev_id, sensor->name, value);
 			continue;
 		}
 
@@ -393,8 +393,9 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 		/* monitor 12V AUX sensor */
 		if (!strcmp(sensor->name, "12V AUX Voltage")) {
 			if (value < AUX_VOLTAGE_WARN) {
-				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
-						sensor->name, value);
+				IFPGA_RAWDEV_PMD_INFO(
+					"%s reach theshold %d mV\n",
+					sensor->name, value);
 				*gsd_start = true;
 				break;
 			}
@@ -1433,6 +1434,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->bus = pci_dev->addr.bus;
+	data->devid = pci_dev->addr.devid;
+	data->function = pci_dev->addr.function;
 	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
-- 
1.8.3.1

[dpdk-dev] [PATCH v13 19/19] raw/ifpga: introducing new irq API

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing new register and unregister API for ifpga interrupt.
1. register FME and AFU interrupt
   ifpga_register_msix_irq()

2. unregister FME and AFU interrupt
   ifpga_unregister_msix_irq()

On PAC N3000 card, there is one PCIe MSIX interrupt for FME managerment,
like the error report, thermal management, we use this interrupt in
ifpga_rawdev device driver. on the other hand, there are about 4 PCIe
MSIX interrupts are reserved for AFU which end-user can use those
interrupts in their AFU logic design. End-user can use those APIs to
register interrupt handler in their AFU drivers.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 105 +++++++++++++++++++++++++++------------
 drivers/raw/ifpga/ifpga_rawdev.h |  14 ++++++
 2 files changed, 88 insertions(+), 31 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 7253644..753dbad 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -78,6 +78,10 @@
 static int ifpga_monitor_start;
 static pthread_t ifpga_monitor_start_thread;
 
+#define IFPGA_MAX_IRQ 12
+/* 0 for FME interrupt, others are reserved for AFU irq */
+static struct rte_intr_handle ifpga_irq_handle[IFPGA_MAX_IRQ];
+
 static struct ifpga_rawdev *
 ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
 static int set_surprise_link_check_aer(
@@ -1331,53 +1335,90 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 	fme_err_handle_catfatal_error(mgr);
 }
 
-static struct rte_intr_handle fme_intr_handle;
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg)
+{
+	struct rte_intr_handle intr_handle;
+
+	if (type == IFPGA_FME_IRQ)
+		intr_handle = ifpga_irq_handle[0];
+	else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+	rte_intr_efd_disable(&intr_handle);
+
+	return rte_intr_callback_unregister(&intr_handle,
+			handler, arg);
+}
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg)
 {
 	int ret;
-	struct fpga_fme_err_irq_set err_irq_set;
+	struct rte_intr_handle intr_handle;
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_accelerator *acc;
 
-	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+	adapter = ifpga_rawdev_get_priv(dev);
+	if (!adapter)
+		return -ENODEV;
 
-	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
-	if (ret)
-		return -EINVAL;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
 
-	fme_intr_handle.fd = fme_intr_handle.efds[0];
+	if (type == IFPGA_FME_IRQ) {
+		intr_handle = ifpga_irq_handle[0];
+		count = 1;
+	} else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
-			fme_intr_handle.vfio_dev_fd,
-			fme_intr_handle.efds[0], fme_intr_handle.fd);
+	intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
 
-	err_irq_set.evtfd = fme_intr_handle.efds[0];
-	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	ret = rte_intr_efd_enable(&intr_handle, count);
 	if (ret)
-		return -EINVAL;
+		return -ENODEV;
+
+	intr_handle.fd = intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("register %s irq, vfio_fd=%d, fd=%d\n",
+			name, intr_handle.vfio_dev_fd,
+			intr_handle.fd);
+
+	if (type == IFPGA_FME_IRQ) {
+		struct fpga_fme_err_irq_set err_irq_set;
+		err_irq_set.evtfd = intr_handle.efds[0];
+
+		ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+		if (ret)
+			return -EINVAL;
+	} else if (type == IFPGA_AFU_IRQ) {
+		acc = opae_adapter_get_acc(adapter, port_id);
+		if (!acc)
+			return -EINVAL;
 
-	/* register FME interrupt using DPDK API */
-	ret = rte_intr_callback_register(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
+		ret = opae_acc_set_irq(acc, vec_start, count, intr_handle.efds);
+		if (ret)
+			return -EINVAL;
+	}
+
+	/* register interrupt handler using DPDK API */
+	ret = rte_intr_callback_register(&intr_handle,
+			handler, (void *)arg);
 	if (ret)
 		return -EINVAL;
 
-	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+	IFPGA_RAWDEV_PMD_INFO("success register %s interrupt\n", name);
 
 	return 0;
 }
 
 static int
-ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
-{
-	rte_intr_efd_disable(&fme_intr_handle);
-
-	return rte_intr_callback_unregister(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
-}
-
-static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -1463,7 +1504,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
-	ret = ifpga_register_fme_interrupt(mgr);
+	ret = ifpga_register_msix_irq(rawdev, 0, IFPGA_FME_IRQ, 0, 0,
+			fme_interrupt_handler, "fme_irq", mgr);
 	if (ret)
 		goto free_adapter_data;
 
@@ -1515,7 +1557,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	if (!mgr)
 		return -ENODEV;
 
-	if (ifpga_unregister_fme_interrupt(mgr))
+	if (ifpga_unregister_msix_irq(IFPGA_FME_IRQ, 0,
+				fme_interrupt_handler, mgr))
 		return -EINVAL;
 
 	opae_adapter_data_free(adapter->data);
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index bd42083..7754beb 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -62,4 +62,18 @@ struct ifpga_rawdev {
 struct ifpga_rawdev *
 ifpga_rawdev_get(const struct rte_rawdev *rawdev);
 
+enum ifpga_irq_type {
+	IFPGA_FME_IRQ = 0,
+	IFPGA_AFU_IRQ = 1,
+};
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg);
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v13 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Ye Xiaolong]

Please fix the patchwork warnings for patch 10/11/12.

Thanks,
Xiaolong

On 10/24, Andy Pei wrote:
>This patch set adds PCIe AER disable and FPGA interrupt support for
>ipn3ke. It also provides a small rework for port bonding between FPGA
>line side port and I40e PF port.
>
>What is the PCI Express AER(Advanced Error Reporting)?
>Advanced Error Reporting capability is implemented with a PCI Express
>advanced error reporting extended capability structure providing more
>robust error reporting. It's also one of PCI Express error reporting
>paradigms. AER is supported by most of PCIe devices.
>
>In PAC N3000 card, some uncertainty errors will cause FPGA reload,
>such as temperature is higher than threshold. From Software point of
>view, FPGA reload means FPGA unplug and plug. For avoiding system
>crash we need to clear AER register before these errors occur.
>
>Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga
>rawdev driver, we implement a FME interrupt function to notify errors
>reported by FME. Besides this, OPAE share code also provide a common
>AFU interrupt API for users to register their own interrupt functions.
>
>v13 updates:
>=========
>- fix meson build issue.
>
>v12 updates:
>=========
>- fix meson build issue.
> 
>v11 updates:
>=========
>- move symbol ifpga_rawdev_ge to the EXPERIMENTAL section of 
>  drivers/raw/ifpga/rte_rawdev_ifpga_version.map
>
>v10 updates:
>=========
>- introducing new irq API
>- fix meson build issue
>
>v9 updates:
>=========
>- Add mutex lock on do_transaction() function for SPI driver to avoid race
>  condition.
>
>v8 updates:
>=========
>- add multiple cards support.
>
>v7 updates:
>==========
>- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
>  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
>- function rte_pmd_i40e_set_switch_dev works as an external API,
>  use port_id but not rte_eth_dev as parameter.
>- add doxygen header here for the new API.
>- update the rte_pmd_i40e_version.map.
>- fix coding style issue.
>- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
>- for functions with a lot of similarity, extract out common function to reduce
>  duplication.
>
>v6 updates:
>=========
>- correct author information.
>- correct typo in commit message and remove Gerrit Change-Id's before
>  submitting upstream
>
>v5 updates:
>==========
>- add lightweight fpga image support. in lightweight fpga image mode,
>  ipn3ke representor will not be probed.
>
>v4 updates:
>===========
>- align with new naming standard.
>
>v3 updates:
>===========
>- Add FPGA network side port MTU configuration
>
>v2 updates:
>===========
>- Add AUX feature support
>
>Andy Pei (2):
>  net/i40e: i40e support ipn3ke FPGA port bonding
>  raw/ifpga: add lightweight fpga image support
>
>Rosen Xu (3):
>  raw/ifpga: add SEU error handler
>  raw/ifpga: add PCIe BDF devices tree scan
>  net/ipn3ke: remove configuration for i40e port bonding
>
>Tianfei zhang (14):
>  raw/ifpga/base: add irq support
>  raw/ifpga/base: clear pending bit
>  raw/ifpga/base: add SEU error support
>  raw/ifpga/base: add device tree support
>  raw/ifpga/base: align the send buffer for SPI
>  raw/ifpga/base: add sensor support
>  raw/ifpga/base: introducing sensor APIs
>  raw/ifpga/base: update SEU register definition
>  raw/ifpga/base: add secure support
>  raw/ifpga/base: configure FEC mode
>  raw/ifpga/base: clean fme errors
>  raw/ifpga/base: add new API get board info
>  raw/ifpga/base: add multiple cards support
>  raw/ifpga: introducing new irq API
>
> config/common_base                             |   4 +-
> config/common_linux                            |   6 +
> drivers/meson.build                            |   6 +-
> drivers/net/i40e/base/i40e_type.h              |   3 +
> drivers/net/i40e/i40e_ethdev.c                 |   6 +
> drivers/net/i40e/rte_pmd_i40e.c                |  21 +
> drivers/net/i40e/rte_pmd_i40e.h                |  18 +
> drivers/net/i40e/rte_pmd_i40e_version.map      |   8 +-
> drivers/net/ipn3ke/Makefile                    |   2 +
> drivers/net/ipn3ke/ipn3ke_ethdev.c             | 289 +-------
> drivers/net/ipn3ke/ipn3ke_representor.c        |   8 +-
> drivers/net/ipn3ke/meson.build                 |  24 +-
> drivers/raw/ifpga/base/ifpga_api.c             |  21 +
> drivers/raw/ifpga/base/ifpga_defines.h         |  75 +-
> drivers/raw/ifpga/base/ifpga_feature_dev.c     |  59 ++
> drivers/raw/ifpga/base/ifpga_feature_dev.h     |   3 +
> drivers/raw/ifpga/base/ifpga_fme.c             | 166 ++++-
> drivers/raw/ifpga/base/ifpga_fme_error.c       |  74 +-
> drivers/raw/ifpga/base/ifpga_hw.h              |   2 +-
> drivers/raw/ifpga/base/ifpga_port.c            |  18 +
> drivers/raw/ifpga/base/ifpga_port_error.c      |  19 +
> drivers/raw/ifpga/base/meson.build             |   2 +-
> drivers/raw/ifpga/base/opae_debug.c            |   3 +
> drivers/raw/ifpga/base/opae_hw_api.c           | 137 ++++
> drivers/raw/ifpga/base/opae_hw_api.h           |  26 +
> drivers/raw/ifpga/base/opae_i2c.c              |  44 +-
> drivers/raw/ifpga/base/opae_i2c.h              |   3 +-
> drivers/raw/ifpga/base/opae_ifpga_hw_api.h     |   2 +
> drivers/raw/ifpga/base/opae_intel_max10.c      | 599 +++++++++++++++-
> drivers/raw/ifpga/base/opae_intel_max10.h      | 157 ++++-
> drivers/raw/ifpga/base/opae_osdep.h            |   7 +-
> drivers/raw/ifpga/base/opae_spi.c              |   5 -
> drivers/raw/ifpga/base/opae_spi.h              |  26 +-
> drivers/raw/ifpga/base/opae_spi_transaction.c  |  84 ++-
> drivers/raw/ifpga/ifpga_rawdev.c               | 903 ++++++++++++++++++++++++-
> drivers/raw/ifpga/ifpga_rawdev.h               |  30 +
> drivers/raw/ifpga/meson.build                  |  25 +-
> drivers/raw/ifpga/rte_rawdev_ifpga_version.map |   3 +
> mk/rte.app.mk                                  |   2 +-
> 39 files changed, 2448 insertions(+), 442 deletions(-)
>
>-- 
>1.8.3.1
>

Re: [dpdk-dev] [PATCH v13 00/19] add PCIe AER disable and IRQ support for ipn3ke

[Pei, Andy]

Hi xiaolong

I think concerning the using rte_exit in driver issue we should have a more detailed discuss with you and Rosen Xu.

-----Original Message-----
From: Ye, Xiaolong 
Sent: Friday, October 25, 2019 10:10 AM
To: Pei, Andy <andy.pei@intel.com>
Cc: dev@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Yigit, Ferruh <ferruh.yigit@intel.com>; Richardson, Bruce <bruce.richardson@intel.com>
Subject: Re: [PATCH v13 00/19] add PCIe AER disable and IRQ support for ipn3ke

Please fix the patchwork warnings for patch 10/11/12.

Thanks,
Xiaolong

On 10/24, Andy Pei wrote:
>This patch set adds PCIe AER disable and FPGA interrupt support for 
>ipn3ke. It also provides a small rework for port bonding between FPGA 
>line side port and I40e PF port.
>
>What is the PCI Express AER(Advanced Error Reporting)?
>Advanced Error Reporting capability is implemented with a PCI Express 
>advanced error reporting extended capability structure providing more 
>robust error reporting. It's also one of PCI Express error reporting 
>paradigms. AER is supported by most of PCIe devices.
>
>In PAC N3000 card, some uncertainty errors will cause FPGA reload, such 
>as temperature is higher than threshold. From Software point of view, 
>FPGA reload means FPGA unplug and plug. For avoiding system crash we 
>need to clear AER register before these errors occur.
>
>Currently PAC N3000 card FME and AFU all provide interrupts, in ifpga 
>rawdev driver, we implement a FME interrupt function to notify errors 
>reported by FME. Besides this, OPAE share code also provide a common 
>AFU interrupt API for users to register their own interrupt functions.
>
>v13 updates:
>=========
>- fix meson build issue.
>
>v12 updates:
>=========
>- fix meson build issue.
> 
>v11 updates:
>=========
>- move symbol ifpga_rawdev_ge to the EXPERIMENTAL section of
>  drivers/raw/ifpga/rte_rawdev_ifpga_version.map
>
>v10 updates:
>=========
>- introducing new irq API
>- fix meson build issue
>
>v9 updates:
>=========
>- Add mutex lock on do_transaction() function for SPI driver to avoid 
>race
>  condition.
>
>v8 updates:
>=========
>- add multiple cards support.
>
>v7 updates:
>==========
>- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
>  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
>- function rte_pmd_i40e_set_switch_dev works as an external API,
>  use port_id but not rte_eth_dev as parameter.
>- add doxygen header here for the new API.
>- update the rte_pmd_i40e_version.map.
>- fix coding style issue.
>- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
>- for functions with a lot of similarity, extract out common function 
>to reduce
>  duplication.
>
>v6 updates:
>=========
>- correct author information.
>- correct typo in commit message and remove Gerrit Change-Id's before
>  submitting upstream
>
>v5 updates:
>==========
>- add lightweight fpga image support. in lightweight fpga image mode,
>  ipn3ke representor will not be probed.
>
>v4 updates:
>===========
>- align with new naming standard.
>
>v3 updates:
>===========
>- Add FPGA network side port MTU configuration
>
>v2 updates:
>===========
>- Add AUX feature support
>
>Andy Pei (2):
>  net/i40e: i40e support ipn3ke FPGA port bonding
>  raw/ifpga: add lightweight fpga image support
>
>Rosen Xu (3):
>  raw/ifpga: add SEU error handler
>  raw/ifpga: add PCIe BDF devices tree scan
>  net/ipn3ke: remove configuration for i40e port bonding
>
>Tianfei zhang (14):
>  raw/ifpga/base: add irq support
>  raw/ifpga/base: clear pending bit
>  raw/ifpga/base: add SEU error support
>  raw/ifpga/base: add device tree support
>  raw/ifpga/base: align the send buffer for SPI
>  raw/ifpga/base: add sensor support
>  raw/ifpga/base: introducing sensor APIs
>  raw/ifpga/base: update SEU register definition
>  raw/ifpga/base: add secure support
>  raw/ifpga/base: configure FEC mode
>  raw/ifpga/base: clean fme errors
>  raw/ifpga/base: add new API get board info
>  raw/ifpga/base: add multiple cards support
>  raw/ifpga: introducing new irq API
>
> config/common_base                             |   4 +-
> config/common_linux                            |   6 +
> drivers/meson.build                            |   6 +-
> drivers/net/i40e/base/i40e_type.h              |   3 +
> drivers/net/i40e/i40e_ethdev.c                 |   6 +
> drivers/net/i40e/rte_pmd_i40e.c                |  21 +
> drivers/net/i40e/rte_pmd_i40e.h                |  18 +
> drivers/net/i40e/rte_pmd_i40e_version.map      |   8 +-
> drivers/net/ipn3ke/Makefile                    |   2 +
> drivers/net/ipn3ke/ipn3ke_ethdev.c             | 289 +-------
> drivers/net/ipn3ke/ipn3ke_representor.c        |   8 +-
> drivers/net/ipn3ke/meson.build                 |  24 +-
> drivers/raw/ifpga/base/ifpga_api.c             |  21 +
> drivers/raw/ifpga/base/ifpga_defines.h         |  75 +-
> drivers/raw/ifpga/base/ifpga_feature_dev.c     |  59 ++
> drivers/raw/ifpga/base/ifpga_feature_dev.h     |   3 +
> drivers/raw/ifpga/base/ifpga_fme.c             | 166 ++++-
> drivers/raw/ifpga/base/ifpga_fme_error.c       |  74 +-
> drivers/raw/ifpga/base/ifpga_hw.h              |   2 +-
> drivers/raw/ifpga/base/ifpga_port.c            |  18 +
> drivers/raw/ifpga/base/ifpga_port_error.c      |  19 +
> drivers/raw/ifpga/base/meson.build             |   2 +-
> drivers/raw/ifpga/base/opae_debug.c            |   3 +
> drivers/raw/ifpga/base/opae_hw_api.c           | 137 ++++
> drivers/raw/ifpga/base/opae_hw_api.h           |  26 +
> drivers/raw/ifpga/base/opae_i2c.c              |  44 +-
> drivers/raw/ifpga/base/opae_i2c.h              |   3 +-
> drivers/raw/ifpga/base/opae_ifpga_hw_api.h     |   2 +
> drivers/raw/ifpga/base/opae_intel_max10.c      | 599 +++++++++++++++-
> drivers/raw/ifpga/base/opae_intel_max10.h      | 157 ++++-
> drivers/raw/ifpga/base/opae_osdep.h            |   7 +-
> drivers/raw/ifpga/base/opae_spi.c              |   5 -
> drivers/raw/ifpga/base/opae_spi.h              |  26 +-
> drivers/raw/ifpga/base/opae_spi_transaction.c  |  84 ++-
> drivers/raw/ifpga/ifpga_rawdev.c               | 903 ++++++++++++++++++++++++-
> drivers/raw/ifpga/ifpga_rawdev.h               |  30 +
> drivers/raw/ifpga/meson.build                  |  25 +-
> drivers/raw/ifpga/rte_rawdev_ifpga_version.map |   3 +
> mk/rte.app.mk                                  |   2 +-
> 39 files changed, 2448 insertions(+), 442 deletions(-)
>
>--
>1.8.3.1
>

[dpdk-dev] [PATCH v12 02/19] raw/ifpga/base: add irq support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME global error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

To build this feature, CONFIG_RTE_EAL_VFIO should be enabled.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                         |  2 +-
 config/common_linux                        |  6 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 59 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 19 ++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 18 +++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 19 ++++++++++
 6 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/config/common_base b/config/common_base
index 2323159..fef2f38 100644
--- a/config/common_base
+++ b/config/common_base
@@ -775,7 +775,7 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
 #
 # Compile PMD for Intel FPGA raw device
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
 
 #
 # Compile PMD for Intel IOAT raw device
diff --git a/config/common_linux b/config/common_linux
index 96e2e1f..a78b8c6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -68,3 +68,9 @@ CONFIG_RTE_LIBRTE_HINIC_PMD=y
 # Hisilicon HNS3 PMD driver
 #
 CONFIG_RTE_LIBRTE_HNS3_PMD=y
+
+#
+# Compile PMD for Intel FPGA raw device
+# To compile, CONFIG_RTE_EAL_VFIO should be enabled.
+#
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..0f852a7 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,61 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+#define FPGA_MAX_MSIX_VEC_COUNT	128
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MAX_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	/* 0 < irq_set->count < FPGA_MAX_MSIX_VEC_COUNT */
+	irq_set->count = count ?
+		(count > FPGA_MAX_MSIX_VEC_COUNT ?
+		 FPGA_MAX_MSIX_VEC_COUNT : count) : 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..2978c79 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,28 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&fme->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..c0aaf01 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,27 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..189f762 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,28 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 03/19] raw/ifpga/base: clear pending bit

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 2978c79..be041ec 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 04/19] raw/ifpga/base: add SEU error support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 31 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index be041ec..5d6d630 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,33 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr(struct ifpga_fme_hw *fme,
+		u64 *val, bool high)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+				FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	if (high)
+		*val = readq(&fme_err->seu_emr_h);
+	else
+		*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +297,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr(fme, &prop->data, 0);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr(fme, &prop->data, 1);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 05/19] raw/ifpga/base: add device tree support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

To eliminate build error, we currently do not compile raw/ifpga
and net/ipn3ke. User should install libfdt and libfdt-devel first,
modify config/common_linux, CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
to CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y, modify config/common_base,
CONFIG_RTE_LIBRTE_IPN3KE_PMD=n to CONFIG_RTE_LIBRTE_IPN3KE_PMD=y.
Then this function can work.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                        |   2 +-
 config/common_linux                       |   2 +-
 drivers/net/ipn3ke/meson.build            |  24 +++-
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 drivers/raw/ifpga/meson.build             |  25 ++--
 mk/rte.app.mk                             |   2 +-
 7 files changed, 231 insertions(+), 17 deletions(-)

diff --git a/config/common_base b/config/common_base
index fef2f38..3c0ea7e 100644
--- a/config/common_base
+++ b/config/common_base
@@ -339,7 +339,7 @@ CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
 #
 # Compile burst-oriented IPN3KE PMD driver
 #
-CONFIG_RTE_LIBRTE_IPN3KE_PMD=y
+CONFIG_RTE_LIBRTE_IPN3KE_PMD=n
 
 #
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
diff --git a/config/common_linux b/config/common_linux
index a78b8c6..c5cf3d6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -73,4 +73,4 @@ CONFIG_RTE_LIBRTE_HNS3_PMD=y
 # Compile PMD for Intel FPGA raw device
 # To compile, CONFIG_RTE_EAL_VFIO should be enabled.
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
diff --git a/drivers/net/ipn3ke/meson.build b/drivers/net/ipn3ke/meson.build
index 74b4d7c..e3c8a67 100644
--- a/drivers/net/ipn3ke/meson.build
+++ b/drivers/net/ipn3ke/meson.build
@@ -8,10 +8,22 @@
 #  rte_eth_dev_destroy()
 #  rte_eth_switch_domain_free()
 #
-allow_experimental_apis = true
 
-sources += files('ipn3ke_ethdev.c',
-	'ipn3ke_representor.c',
-	'ipn3ke_tm.c',
-	'ipn3ke_flow.c')
-deps += ['bus_ifpga', 'sched']
+dep = dependency('libfdt', required: false)
+if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
+	build = false
+	reason = 'missing dependency, "libfdt"'
+endif
+
+if build
+	allow_experimental_apis = true
+
+	sources += files('ipn3ke_ethdev.c',
+		'ipn3ke_representor.c',
+		'ipn3ke_tm.c',
+		'ipn3ke_flow.c')
+	deps += ['bus_ifpga', 'sched', 'pmd_i40e', 'rawdev', 'rawdev_ifpga']
+endif
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
index 0ab6fd7..69debff 100644
--- a/drivers/raw/ifpga/meson.build
+++ b/drivers/raw/ifpga/meson.build
@@ -3,18 +3,27 @@
 
 version = 1
 
-subdir('base')
-objs = [base_objs]
-
 dep = dependency('libfdt', required: false)
 if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
 	build = false
 	reason = 'missing dependency, "libfdt"'
 endif
-deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
-	'bus_vdev', 'bus_ifpga', 'net']
-sources = files('ifpga_rawdev.c')
 
-includes += include_directories('base')
+if build
+	subdir('base')
+	objs = [base_objs]
+
+	deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
+		'bus_vdev', 'bus_ifpga', 'net']
+	ext_deps += dep
 
-allow_experimental_apis = true
+	sources = files('ifpga_rawdev.c')
+
+	includes += include_directories('base')
+	includes += include_directories('../../net/ipn3ke')
+
+	allow_experimental_apis = true
+endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index b91273f..28f3ef3 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -321,7 +321,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 06/19] raw/ifpga/base: align the send buffer for SPI

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 07/19] raw/ifpga/base: add sensor support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 08/19] raw/ifpga/base: introducing sensor APIs

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 09/19] raw/ifpga/base: update SEU register definition

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 10/19] raw/ifpga: add SEU error handler

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 245 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 1825143..f5e6119 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -27,6 +27,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -605,6 +607,236 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -652,6 +884,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -677,6 +910,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -696,6 +933,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -720,6 +958,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 11/19] raw/ifpga: add PCIe BDF devices tree scan

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 551 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 562 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index f5e6119..01ff76a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_malloc.h>
@@ -17,7 +19,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -25,6 +27,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -37,6 +40,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -64,6 +73,494 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(
+		struct ifpga_rawdev *ifpga_rdev, int force_disable);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+		int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
+					sensor->name, value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(
+	struct ifpga_rawdev *ifpga_rdev, int force_disable)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable || force_disable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev,
+					gsd_enable);
+				if (ret == 1 && !gsd_enable) {
+					gsd_enable = 1;
+					i = -1;
+				}
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -372,8 +869,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -842,6 +1340,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -855,7 +1354,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -869,6 +1368,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -987,6 +1494,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1018,13 +1526,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1038,7 +1565,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1065,6 +1593,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 12/19] net/ipn3ke: remove configuration for i40e port bonding

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/ipn3ke/Makefile             |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c      | 289 ++++----------------------------
 drivers/net/ipn3ke/ipn3ke_representor.c |   8 +-
 3 files changed, 44 insertions(+), 255 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..2c65e49 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index 28d8aaf..3051cdf 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -324,7 +325,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -342,6 +344,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -409,9 +417,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -462,7 +467,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -489,7 +498,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -501,6 +515,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -508,6 +538,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -553,254 +584,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index d37f5e2..7e5d29d 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2918,8 +2919,11 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		rte_pmd_i40e_set_switch_dev(rpst->i40e_pf_eth_port_id,
+					    rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 13/19] raw/ifpga/base: add secure support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 137 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 ++++++++++++-----
 4 files changed, 198 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..748ab56 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,63 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager")) {
+			sysmgr_offset = offset;
+			break;
+		}
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +576,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 14/19] raw/ifpga/base: configure FEC mode

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 15/19] raw/ifpga/base: clean fme errors

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occurred.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 5d6d630..5905eac 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 01ff76a..95f079a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1174,6 +1174,25 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1183,6 +1202,9 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 16/19] raw/ifpga/base: add new API get board info

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..9f0147d 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disaggregate;
+	u8 lightweight;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..799d67d 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweight = id.lightweiht;
+	fme->board_info.disaggregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweight:%s\n"
+			"disaggregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweight),
+			check_support(fme->board_info.disaggregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 17/19] raw/ifpga: add lightweight fpga image support

[Andy Pei]

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 95f079a..e87af66 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -835,6 +835,8 @@ static int set_surprise_link_check_aer(
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -861,22 +863,40 @@ static int set_surprise_link_check_aer(
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweight) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
-		uuid.b + 8, sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 18/19] raw/ifpga/base: add multiple cards support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, there is one MAX10 chip in each card, and
all of the sensors are connected to MAX10 chip. To support multiple
cards in one server, we introducing a sensor device list under
intel_max10_device instead of a global list. On the other hand, we
using separate intel_max10_device instance for each opae_adatper.

Add mutex lock on do_transaction() function for SPI driver to avoid
race condition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c            |  40 +++++++---
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          |  14 ++--
 drivers/raw/ifpga/base/opae_hw_api.h          |  15 ++--
 drivers/raw/ifpga/base/opae_i2c.c             |  44 ++++++++---
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_intel_max10.c     | 110 ++++++++++++++------------
 drivers/raw/ifpga/base/opae_intel_max10.h     |  19 +++--
 drivers/raw/ifpga/base/opae_spi.c             |   5 --
 drivers/raw/ifpga/base/opae_spi.h             |   3 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  44 +++++++++--
 drivers/raw/ifpga/ifpga_rawdev.c              |  14 ++--
 12 files changed, 205 insertions(+), 109 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 799d67d..c31a94c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -839,8 +839,13 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	struct ifpga_hw *hw;
 	u32 val;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
@@ -854,7 +859,10 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	fme->board_info.seu = id.seu;
 	fme->board_info.ptp = id.ptp;
 
-	dev_info(fme, "found: board: %s type: %s\n",
+	dev_info(fme, "found: PCI dev: %02x:%02x:%x board: %s type: %s\n",
+			hw->pci_data->bus,
+			hw->pci_data->devid,
+			hw->pci_data->function,
 			board_major_to_string(fme->board_info.major),
 			board_type_to_string(fme->board_info.type));
 
@@ -882,11 +890,11 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
-	if (max10_sys_read(MAX10_BUILD_VER, &val))
+	if (max10_sys_read(fme->max10_dev, MAX10_BUILD_VER, &val))
 		return -EINVAL;
 	fme->board_info.max10_version = val & 0xffffff;
 
-	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+	if (max10_sys_read(fme->max10_dev, NIOS2_FW_VERSION, &val))
 		return -EINVAL;
 	fme->board_info.nios_fw_version = val & 0xffffff;
 
@@ -897,12 +905,12 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	return 0;
 }
 
-static int spi_self_checking(void)
+static int spi_self_checking(struct intel_max10_device *dev)
 {
 	u32 val;
 	int ret;
 
-	ret = max10_sys_read(MAX10_TEST_REG, &val);
+	ret = max10_sys_read(dev, MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
@@ -937,10 +945,11 @@ static int fme_spi_init(struct ifpga_feature *feature)
 		goto spi_fail;
 	}
 
+
 	fme->max10_dev = max10;
 
 	/* SPI self test */
-	if (spi_self_checking()) {
+	if (spi_self_checking(max10)) {
 		ret = -EIO;
 		goto max10_fail;
 	}
@@ -1041,8 +1050,18 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 	struct ifpga_fme_hw *fme = (struct ifpga_fme_hw *)feature->parent;
 	struct altera_spi_device *spi_master;
 	struct intel_max10_device *max10;
+	struct ifpga_hw *hw;
+	struct opae_manager *mgr;
 	int ret = 0;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
+	mgr = hw->adapter->mgr;
+	if (!mgr)
+		return -ENODEV;
+
 	dev_info(fme, "FME SPI Master (NIOS) Init.\n");
 	dev_debug(fme, "FME SPI base addr %p.\n",
 			feature->addr);
@@ -1080,12 +1099,15 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 		goto release_dev;
 	}
 
+	max10->bus = hw->pci_data->bus;
+
 	fme_get_board_interface(fme);
 
 	fme->max10_dev = max10;
+	mgr->sensor_list = &max10->opae_sensor_list;
 
 	/* SPI self test */
-	if (spi_self_checking())
+	if (spi_self_checking(max10))
 		goto spi_fail;
 
 	return ret;
@@ -1344,7 +1366,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(dev, PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1372,7 +1394,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(sensor->value_reg, value)) {
+	if (max10_sys_read(dev, sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_debug.c b/drivers/raw/ifpga/base/opae_debug.c
index 88f2d5c..dad3ea3 100644
--- a/drivers/raw/ifpga/base/opae_debug.c
+++ b/drivers/raw/ifpga/base/opae_debug.c
@@ -59,6 +59,9 @@ static void opae_adapter_data_dump(void *data)
 		opae_log("OPAE Adapter Type = PCI\n");
 		opae_log("PCI Device ID: 0x%04x\n", d_pci->device_id);
 		opae_log("PCI Vendor ID: 0x%04x\n", d_pci->vendor_id);
+		opae_log("PCI bus: 0x%04x\n", d_pci->bus);
+		opae_log("PCI devid: 0x%04x\n", d_pci->devid);
+		opae_log("PCI function: 0x%04x\n", d_pci->function);
 
 		for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 			r = &d_pci->region[i];
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 1ccc967..c969dfe 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -583,11 +583,12 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_id(unsigned int id)
+opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (sensor->id == id)
 			return sensor;
 
@@ -601,11 +602,12 @@ struct opae_sensor_info *
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_name(const char *name)
+opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (!strcmp(sensor->name, name))
 			return sensor;
 
@@ -630,7 +632,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_name(name);
+	sensor = opae_mgr_get_sensor_by_name(mgr, name);
 	if (!sensor)
 		return -ENODEV;
 
@@ -658,7 +660,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_id(id);
+	sensor = opae_mgr_get_sensor_by_id(mgr, id);
 	if (!sensor)
 		return -ENODEV;
 
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index b78fbd5..cdf369f 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -40,6 +40,7 @@ struct opae_manager {
 	struct opae_adapter *adapter;
 	struct opae_manager_ops *ops;
 	struct opae_manager_networking_ops *network_ops;
+	struct opae_sensor_list *sensor_list;
 	void *data;
 };
 
@@ -75,9 +76,8 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
-extern struct opae_sensor_list opae_sensor_list;
-#define opae_mgr_for_each_sensor(sensor) \
-	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+#define opae_mgr_for_each_sensor(mgr, sensor) \
+	TAILQ_FOREACH(sensor, mgr->sensor_list, node)
 
 /* OPAE Manager APIs */
 struct opae_manager *
@@ -88,8 +88,10 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
-struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
-struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id);
 int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
 		const char *name, unsigned int *value);
 int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
@@ -241,6 +243,9 @@ struct opae_adapter_data_pci {
 	enum opae_adapter_type type;
 	u16 device_id;
 	u16 vendor_id;
+	u16 bus; /*Device bus for PCI */
+	u16 devid; /* Device ID */
+	u16 function; /* Device function */
 	struct opae_reg_region region[PCI_MAX_RESOURCE];
 	int vfio_dev_fd;  /* VFIO device file descriptor */
 };
diff --git a/drivers/raw/ifpga/base/opae_i2c.c b/drivers/raw/ifpga/base/opae_i2c.c
index f8bc247..846d751 100644
--- a/drivers/raw/ifpga/base/opae_i2c.c
+++ b/drivers/raw/ifpga/base/opae_i2c.c
@@ -28,6 +28,9 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 {
 	u8 msgbuf[2];
 	int i = 0;
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
 
 	if (flags & I2C_FLAG_ADDR16)
 		msgbuf[i++] = offset >> 8;
@@ -49,10 +52,16 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		},
 	};
 
-	if (!dev->xfer)
-		return -ENODEV;
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	ret = i2c_transfer(dev, msg, 2);
 
-	return i2c_transfer(dev, msg, 2);
+exit:
+	pthread_mutex_unlock(&dev->lock);
+	return ret;
 }
 
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
@@ -63,12 +72,18 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	int ret;
 	int i = 0;
 
-	if (!dev->xfer)
-		return -ENODEV;
+	pthread_mutex_lock(&dev->lock);
+
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
 
 	buf = opae_malloc(I2C_MAX_OFFSET_LEN + len);
-	if (!buf)
-		return -ENOMEM;
+	if (!buf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
 
 	msg.addr = slave_addr;
 	msg.flags = 0;
@@ -84,6 +99,8 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	ret = i2c_transfer(dev, &msg, 1);
 
 	opae_free(buf);
+exit:
+	pthread_mutex_unlock(&dev->lock);
 	return ret;
 }
 
@@ -477,14 +494,19 @@ struct altera_i2c_dev *altera_i2c_probe(void *base)
 	dev->i2c_clk = dev->i2c_param.ref_clk * 1000000;
 	dev->xfer = altera_i2c_xfer;
 
+	if (pthread_mutex_init(&dev->lock, NULL))
+		return NULL;
+
 	altera_i2c_hardware_init(dev);
 
 	return dev;
 }
 
-int altera_i2c_remove(struct altera_i2c_dev *dev)
+void altera_i2c_remove(struct altera_i2c_dev *dev)
 {
-	altera_i2c_disable(dev);
-
-	return 0;
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
+		altera_i2c_disable(dev);
+		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/base/opae_i2c.h b/drivers/raw/ifpga/base/opae_i2c.h
index 8890c8f..266e127 100644
--- a/drivers/raw/ifpga/base/opae_i2c.h
+++ b/drivers/raw/ifpga/base/opae_i2c.h
@@ -93,6 +93,7 @@ struct altera_i2c_dev {
 	u32 isr_mask;
 	u8 *buf;
 	int (*xfer)(struct altera_i2c_dev *dev, struct i2c_msg *msg, int num);
+	pthread_mutex_t lock;
 };
 
 /**
@@ -114,7 +115,7 @@ enum i2c_msg_flags {
 };
 
 struct altera_i2c_dev *altera_i2c_probe(void *base);
-int altera_i2c_remove(struct altera_i2c_dev *dev);
+void altera_i2c_remove(struct altera_i2c_dev *dev);
 int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		u32 offset, u8 *buf, u32 count);
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 748ab56..8e23ca1 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -5,45 +5,50 @@
 #include "opae_intel_max10.h"
 #include <libfdt.h>
 
-static struct intel_max10_device *g_max10;
-
-struct opae_sensor_list opae_sensor_list =
-	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
-
-int max10_reg_read(unsigned int reg, unsigned int *val)
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_read(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x\n", __func__, dev->bus, reg);
+
+	return spi_transaction_read(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)val);
 }
 
-int max10_reg_write(unsigned int reg, unsigned int val)
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val)
 {
 	unsigned int tmp = val;
 
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_write(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x, val:0x%x\n", __func__,
+			dev->bus, reg, val);
+
+	return spi_transaction_write(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)&tmp);
 }
 
-int max10_sys_read(unsigned int offset, unsigned int *val)
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_read(g_max10->base + offset, val);
+
+	return max10_reg_read(dev, dev->base + offset, val);
 }
 
-int max10_sys_write(unsigned int offset, unsigned int val)
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_write(g_max10->base + offset, val);
+	return max10_reg_write(dev, dev->base + offset, val);
 }
 
 static struct max10_compatible_id max10_id_table[] = {
@@ -86,8 +91,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
-static int altera_nor_flash_read(u32 offset,
-		void *buffer, u32 len)
+static int altera_nor_flash_read(struct intel_max10_device *dev,
+	u32 offset, void *buffer, u32 len)
 {
 	int word_len;
 	int i;
@@ -95,13 +100,13 @@ static int altera_nor_flash_read(u32 offset,
 	unsigned int value;
 	int ret;
 
-	if (!buffer || len <= 0)
+	if (!dev || !buffer || len <= 0)
 		return -ENODEV;
 
 	word_len = len/4;
 
 	for (i = 0; i < word_len; i++) {
-		ret = max10_reg_read(offset + i*4,
+		ret = max10_reg_read(dev, offset + i*4,
 				&value);
 		if (ret)
 			return -EBUSY;
@@ -112,12 +117,12 @@ static int altera_nor_flash_read(u32 offset,
 	return 0;
 }
 
-static int enable_nor_flash(bool on)
+static int enable_nor_flash(struct intel_max10_device *dev, bool on)
 {
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_sys_read(RSU_REG, &val);
+	ret = max10_sys_read(dev, RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -128,7 +133,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_sys_write(RSU_REG, val);
+	return max10_sys_write(dev, RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -140,7 +145,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_sys_read(DT_AVAIL_REG, &val);
+	ret = max10_sys_read(max10, DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -151,19 +156,19 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
+	ret = max10_sys_read(max10, DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
 	}
 
-	ret = enable_nor_flash(true);
+	ret = enable_nor_flash(max10, true);
 	if (ret) {
 		dev_err(max10 "fail to enable flash\n");
 		return ret;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	ret = altera_nor_flash_read(max10, dt_addr, &hdr, sizeof(hdr));
 	if (ret) {
 		dev_err(max10 "read fdt header fail\n");
 		goto done;
@@ -188,7 +193,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		goto done;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	ret = altera_nor_flash_read(max10, dt_addr, fdt_root, dt_size);
 	if (ret) {
 		dev_err(max10 "cannot read device table\n");
 		goto done;
@@ -207,7 +212,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	max10->fdt_root = fdt_root;
 
 done:
-	ret = enable_nor_flash(false);
+	ret = enable_nor_flash(max10, false);
 
 	if (ret && fdt_root)
 		opae_free(fdt_root);
@@ -298,12 +303,12 @@ static int fdt_get_named_reg(const void *fdt, int node, const char *name,
 	return fdt_get_reg(fdt, node, idx, start, size);
 }
 
-static void max10_sensor_uinit(void)
+static void max10_sensor_uinit(struct intel_max10_device *dev)
 {
 	struct opae_sensor_info *info;
 
-	TAILQ_FOREACH(info, &opae_sensor_list, node) {
-		TAILQ_REMOVE(&opae_sensor_list, info, node);
+	TAILQ_FOREACH(info, &dev->opae_sensor_list, node) {
+		TAILQ_REMOVE(&dev->opae_sensor_list, info, node);
 		opae_free(info);
 	}
 }
@@ -313,8 +318,8 @@ static bool sensor_reg_valid(struct sensor_reg *reg)
 	return !!reg->size;
 }
 
-static int max10_add_sensor(struct raw_sensor_info *info,
-		struct opae_sensor_info *sensor)
+static int max10_add_sensor(struct intel_max10_device *dev,
+	struct raw_sensor_info *info, struct opae_sensor_info *sensor)
 {
 	int i;
 	int ret = 0;
@@ -332,12 +337,15 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_sys_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(dev, info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
-		if (val == 0xdeadbeef)
+		if (val == 0xdeadbeef) {
+			dev_debug(dev, "%s: sensor:%s invalid 0x%x at:%d\n",
+				__func__, sensor->name, val, i);
 			continue;
+		}
 
 		val *= info->multiplier;
 
@@ -458,7 +466,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
 		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
 
-		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+		dev_debug(dev, "found sensor from DTB: %s: %s: %u: %u\n",
 				raw->name, raw->type,
 				raw->id, raw->multiplier);
 
@@ -473,15 +481,16 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 			goto free_sensor;
 		}
 
-		if (max10_add_sensor(raw, sensor)) {
+		if (max10_add_sensor(dev, raw, sensor)) {
 			ret = -EINVAL;
 			opae_free(sensor);
 			goto free_sensor;
 		}
 
-		if (sensor->flags & OPAE_SENSOR_VALID)
-			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
-		else
+		if (sensor->flags & OPAE_SENSOR_VALID) {
+			TAILQ_INSERT_TAIL(&dev->opae_sensor_list, sensor, node);
+			dev_info(dev, "found valid sensor: %s\n", sensor->name);
+		} else
 			opae_free(sensor);
 
 		opae_free(raw);
@@ -492,7 +501,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 free_sensor:
 	if (raw)
 		opae_free(raw);
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 	return ret;
 }
 
@@ -500,7 +509,7 @@ static int check_max10_version(struct intel_max10_device *dev)
 {
 	unsigned int v;
 
-	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+	if (!max10_reg_read(dev, MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
 				&v)) {
 		if (v != 0xffffffff) {
 			dev_info(dev, "secure MAX10 detected\n");
@@ -565,6 +574,8 @@ struct intel_max10_device *
 	if (!dev)
 		return NULL;
 
+	TAILQ_INIT(&dev->opae_sensor_list);
+
 	dev->spi_master = spi;
 
 	dev->spi_tran_dev = spi_transaction_init(spi, chipselect);
@@ -573,9 +584,6 @@ struct intel_max10_device *
 		goto free_dev;
 	}
 
-	/* set the max10 device firstly */
-	g_max10 = dev;
-
 	/* check the max10 version */
 	ret = check_max10_version(dev);
 	if (ret) {
@@ -601,7 +609,7 @@ struct intel_max10_device *
 	}
 
 	/* read FPGA loading information */
-	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
+	ret = max10_sys_read(dev, FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
 		goto release_max10_hw;
@@ -611,14 +619,13 @@ struct intel_max10_device *
 	return dev;
 
 release_max10_hw:
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return NULL;
@@ -629,7 +636,7 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
@@ -637,7 +644,6 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index e632941..123cdc4 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -26,6 +26,9 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SECURE		BIT(6)
 #define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
@@ -33,6 +36,8 @@ struct intel_max10_device {
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
 	unsigned int base; /* max10 base address */
+	u16 bus;
+	struct opae_sensor_list opae_sensor_list;
 };
 
 /* retimer speed */
@@ -136,17 +141,19 @@ struct opae_retimer_status {
 
 #define DFT_MAX_SIZE		0x7e0000
 
-int max10_reg_read(unsigned int reg, unsigned int *val);
-int max10_reg_write(unsigned int reg, unsigned int val);
-int max10_sys_read(unsigned int offset, unsigned int *val);
-int max10_sys_write(unsigned int offset, unsigned int val);
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val);
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val);
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val);
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
-/** List of opae sensors */
-TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
 
 #define SENSOR_REG_VALUE 0x0
 #define SENSOR_REG_HIGH_WARN 0x1
diff --git a/drivers/raw/ifpga/base/opae_spi.c b/drivers/raw/ifpga/base/opae_spi.c
index cc52782..bfdc83e 100644
--- a/drivers/raw/ifpga/base/opae_spi.c
+++ b/drivers/raw/ifpga/base/opae_spi.c
@@ -181,7 +181,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 	u32 rxd;
 	unsigned int tx_data;
 	u32 status;
-	int retry = 0;
 	int ret;
 
 	while (count < dev->len) {
@@ -194,10 +193,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 				return -EIO;
 			if (status & ALTERA_SPI_STATUS_RRDY_MSK)
 				break;
-			if (retry++ > SPI_MAX_RETRY) {
-				dev_err(dev, "%s, read timeout\n", __func__);
-				return -EBUSY;
-			}
 		}
 
 		ret = spi_reg_read(dev, ALTERA_SPI_RXDATA, &rxd);
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index 6355deb..d20a4c3 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -38,7 +38,7 @@
 #define SPI_WRITE 0x20
 #define WRITE_DATA_MASK GENMASK_ULL(31, 0)
 
-#define SPI_MAX_RETRY 100000
+#define SPI_MAX_RETRY 1000000
 
 #define TYPE_SPI 0
 #define TYPE_NIOS_SPI 1
@@ -102,6 +102,7 @@ struct spi_transaction_dev {
 	struct altera_spi_device *dev;
 	int chipselect;
 	struct spi_tran_buffer *buffer;
+	pthread_mutex_t lock;
 };
 
 struct spi_tran_header {
diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 06ca625..013efee 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -154,6 +154,8 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	unsigned int rx_len = 0;
 	int retry = 0;
 	int spi_flags;
+	unsigned long timeout = msecs_to_timer_cycles(1000);
+	unsigned long ticks;
 	unsigned int resp_max_len = 2 * resp_len;
 
 	print_buffer("before bytes:", send_data, send_len);
@@ -207,8 +209,11 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	if (ret != SPI_FOUND_EOP) {
 		tx_buffer = NULL;
 		tx_len = 0;
-		if (retry++ > 10) {
-			dev_err(NULL, "cannot found valid data from SPI\n");
+		ticks = rte_get_timer_cycles();
+		if (time_after(ticks, timeout) &&
+				retry++ > SPI_MAX_RETRY) {
+			dev_err(NULL, "Have retry %d, found invalid packet data\n",
+				retry);
 			return -EBUSY;
 		}
 
@@ -427,23 +432,36 @@ static int do_transaction(struct spi_transaction_dev *dev, unsigned int addr,
 int spi_transaction_read(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_READ : SPI_TRAN_NON_SEQ_READ);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 int spi_transaction_write(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_WRITE : SPI_TRAN_NON_SEQ_WRITE);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 		int chipselect)
 {
 	struct spi_transaction_dev *spi_tran_dev;
+	int ret;
 
 	spi_tran_dev = opae_malloc(sizeof(struct spi_transaction_dev));
 	if (!spi_tran_dev)
@@ -453,18 +471,28 @@ struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 	spi_tran_dev->chipselect = chipselect;
 
 	spi_tran_dev->buffer = opae_malloc(sizeof(struct spi_tran_buffer));
-	if (!spi_tran_dev->buffer) {
-		opae_free(spi_tran_dev);
-		return NULL;
+	if (!spi_tran_dev->buffer)
+		goto err;
+
+	ret = pthread_mutex_init(&spi_tran_dev->lock, NULL);
+	if (ret) {
+		dev_err(spi_tran_dev, "fail to init mutex lock\n");
+		goto err;
 	}
 
 	return spi_tran_dev;
+
+err:
+	opae_free(spi_tran_dev);
+	return NULL;
 }
 
 void spi_transaction_remove(struct spi_transaction_dev *dev)
 {
 	if (dev && dev->buffer)
 		opae_free(dev->buffer);
-	if (dev)
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
 		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index e87af66..7253644 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -360,7 +360,7 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 	if (!mgr)
 		return -ENODEV;
 
-	opae_mgr_for_each_sensor(sensor) {
+	opae_mgr_for_each_sensor(mgr, sensor) {
 		if (!(sensor->flags & OPAE_SENSOR_VALID))
 			goto fail;
 
@@ -369,8 +369,8 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 			goto fail;
 
 		if (value == 0xdeadbeef) {
-			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
-					sensor->name, value);
+			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value %x\n",
+					raw_dev->dev_id, sensor->name, value);
 			continue;
 		}
 
@@ -393,8 +393,9 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 		/* monitor 12V AUX sensor */
 		if (!strcmp(sensor->name, "12V AUX Voltage")) {
 			if (value < AUX_VOLTAGE_WARN) {
-				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
-						sensor->name, value);
+				IFPGA_RAWDEV_PMD_INFO(
+					"%s reach theshold %d mV\n",
+					sensor->name, value);
 				*gsd_start = true;
 				break;
 			}
@@ -1433,6 +1434,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->bus = pci_dev->addr.bus;
+	data->devid = pci_dev->addr.devid;
+	data->function = pci_dev->addr.function;
 	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
-- 
1.8.3.1

[dpdk-dev] [PATCH v12 19/19] raw/ifpga: introducing new irq API

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing new register and unregister API for ifpga interrupt.
1. register FME and AFU interrupt
   ifpga_register_msix_irq()

2. unregister FME and AFU interrupt
   ifpga_unregister_msix_irq()

On PAC N3000 card, there is one PCIe MSIX interrupt for FME managerment,
like the error report, thermal management, we use this interrupt in
ifpga_rawdev device driver. on the other hand, there are about 4 PCIe
MSIX interrupts are reserved for AFU which end-user can use those
interrupts in their AFU logic design. End-user can use those APIs to
register interrupt handler in their AFU drivers.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 105 +++++++++++++++++++++++++++------------
 drivers/raw/ifpga/ifpga_rawdev.h |  14 ++++++
 2 files changed, 88 insertions(+), 31 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 7253644..753dbad 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -78,6 +78,10 @@
 static int ifpga_monitor_start;
 static pthread_t ifpga_monitor_start_thread;
 
+#define IFPGA_MAX_IRQ 12
+/* 0 for FME interrupt, others are reserved for AFU irq */
+static struct rte_intr_handle ifpga_irq_handle[IFPGA_MAX_IRQ];
+
 static struct ifpga_rawdev *
 ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
 static int set_surprise_link_check_aer(
@@ -1331,53 +1335,90 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 	fme_err_handle_catfatal_error(mgr);
 }
 
-static struct rte_intr_handle fme_intr_handle;
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg)
+{
+	struct rte_intr_handle intr_handle;
+
+	if (type == IFPGA_FME_IRQ)
+		intr_handle = ifpga_irq_handle[0];
+	else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+	rte_intr_efd_disable(&intr_handle);
+
+	return rte_intr_callback_unregister(&intr_handle,
+			handler, arg);
+}
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg)
 {
 	int ret;
-	struct fpga_fme_err_irq_set err_irq_set;
+	struct rte_intr_handle intr_handle;
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_accelerator *acc;
 
-	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+	adapter = ifpga_rawdev_get_priv(dev);
+	if (!adapter)
+		return -ENODEV;
 
-	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
-	if (ret)
-		return -EINVAL;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
 
-	fme_intr_handle.fd = fme_intr_handle.efds[0];
+	if (type == IFPGA_FME_IRQ) {
+		intr_handle = ifpga_irq_handle[0];
+		count = 1;
+	} else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
-			fme_intr_handle.vfio_dev_fd,
-			fme_intr_handle.efds[0], fme_intr_handle.fd);
+	intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
 
-	err_irq_set.evtfd = fme_intr_handle.efds[0];
-	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	ret = rte_intr_efd_enable(&intr_handle, count);
 	if (ret)
-		return -EINVAL;
+		return -ENODEV;
+
+	intr_handle.fd = intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("register %s irq, vfio_fd=%d, fd=%d\n",
+			name, intr_handle.vfio_dev_fd,
+			intr_handle.fd);
+
+	if (type == IFPGA_FME_IRQ) {
+		struct fpga_fme_err_irq_set err_irq_set;
+		err_irq_set.evtfd = intr_handle.efds[0];
+
+		ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+		if (ret)
+			return -EINVAL;
+	} else if (type == IFPGA_AFU_IRQ) {
+		acc = opae_adapter_get_acc(adapter, port_id);
+		if (!acc)
+			return -EINVAL;
 
-	/* register FME interrupt using DPDK API */
-	ret = rte_intr_callback_register(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
+		ret = opae_acc_set_irq(acc, vec_start, count, intr_handle.efds);
+		if (ret)
+			return -EINVAL;
+	}
+
+	/* register interrupt handler using DPDK API */
+	ret = rte_intr_callback_register(&intr_handle,
+			handler, (void *)arg);
 	if (ret)
 		return -EINVAL;
 
-	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+	IFPGA_RAWDEV_PMD_INFO("success register %s interrupt\n", name);
 
 	return 0;
 }
 
 static int
-ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
-{
-	rte_intr_efd_disable(&fme_intr_handle);
-
-	return rte_intr_callback_unregister(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
-}
-
-static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -1463,7 +1504,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
-	ret = ifpga_register_fme_interrupt(mgr);
+	ret = ifpga_register_msix_irq(rawdev, 0, IFPGA_FME_IRQ, 0, 0,
+			fme_interrupt_handler, "fme_irq", mgr);
 	if (ret)
 		goto free_adapter_data;
 
@@ -1515,7 +1557,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	if (!mgr)
 		return -ENODEV;
 
-	if (ifpga_unregister_fme_interrupt(mgr))
+	if (ifpga_unregister_msix_irq(IFPGA_FME_IRQ, 0,
+				fme_interrupt_handler, mgr))
 		return -EINVAL;
 
 	opae_adapter_data_free(adapter->data);
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index bd42083..7754beb 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -62,4 +62,18 @@ struct ifpga_rawdev {
 struct ifpga_rawdev *
 ifpga_rawdev_get(const struct rte_rawdev *rawdev);
 
+enum ifpga_irq_type {
+	IFPGA_FME_IRQ = 0,
+	IFPGA_AFU_IRQ = 1,
+};
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg);
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 02/19] raw/ifpga/base: add irq support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME global error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

To build this feature, CONFIG_RTE_EAL_VFIO should be enabled.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                         |  2 +-
 config/common_linux                        |  6 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 59 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 19 ++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 18 +++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 19 ++++++++++
 6 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/config/common_base b/config/common_base
index e843a21..68a4f70 100644
--- a/config/common_base
+++ b/config/common_base
@@ -772,7 +772,7 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
 #
 # Compile PMD for Intel FPGA raw device
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
 
 #
 # Compile PMD for Intel IOAT raw device
diff --git a/config/common_linux b/config/common_linux
index 96e2e1f..a78b8c6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -68,3 +68,9 @@ CONFIG_RTE_LIBRTE_HINIC_PMD=y
 # Hisilicon HNS3 PMD driver
 #
 CONFIG_RTE_LIBRTE_HNS3_PMD=y
+
+#
+# Compile PMD for Intel FPGA raw device
+# To compile, CONFIG_RTE_EAL_VFIO should be enabled.
+#
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..0f852a7 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,61 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+#define FPGA_MAX_MSIX_VEC_COUNT	128
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MAX_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	/* 0 < irq_set->count < FPGA_MAX_MSIX_VEC_COUNT */
+	irq_set->count = count ?
+		(count > FPGA_MAX_MSIX_VEC_COUNT ?
+		 FPGA_MAX_MSIX_VEC_COUNT : count) : 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..2978c79 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,28 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&fme->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..c0aaf01 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,27 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..189f762 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,28 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 03/19] raw/ifpga/base: clear pending bit

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 2978c79..be041ec 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 04/19] raw/ifpga/base: add SEU error support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 31 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index be041ec..5d6d630 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,33 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr(struct ifpga_fme_hw *fme,
+		u64 *val, bool high)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+				FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	if (high)
+		*val = readq(&fme_err->seu_emr_h);
+	else
+		*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +297,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr(fme, &prop->data, 0);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr(fme, &prop->data, 1);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 05/19] raw/ifpga/base: add device tree support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

To eliminate build error, we currently do not compile raw/ifpga
and net/ipn3ke. User should install libfdt and libfdt-devel first,
modify config/common_linux, CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
to CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y, modify config/common_base,
CONFIG_RTE_LIBRTE_IPN3KE_PMD=n to CONFIG_RTE_LIBRTE_IPN3KE_PMD=y.
Then this function can work.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                        |   2 +-
 config/common_linux                       |   2 +-
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 mk/rte.app.mk                             |   2 +-
 5 files changed, 196 insertions(+), 3 deletions(-)

diff --git a/config/common_base b/config/common_base
index 68a4f70..c0c9d5e 100644
--- a/config/common_base
+++ b/config/common_base
@@ -336,7 +336,7 @@ CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
 #
 # Compile burst-oriented IPN3KE PMD driver
 #
-CONFIG_RTE_LIBRTE_IPN3KE_PMD=y
+CONFIG_RTE_LIBRTE_IPN3KE_PMD=n
 
 #
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
diff --git a/config/common_linux b/config/common_linux
index a78b8c6..c5cf3d6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -73,4 +73,4 @@ CONFIG_RTE_LIBRTE_HNS3_PMD=y
 # Compile PMD for Intel FPGA raw device
 # To compile, CONFIG_RTE_EAL_VFIO should be enabled.
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index b91273f..28f3ef3 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -321,7 +321,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 06/19] raw/ifpga/base: align the send buffer for SPI

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 07/19] raw/ifpga/base: add sensor support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 08/19] raw/ifpga/base: introducing sensor APIs

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 09/19] raw/ifpga/base: update SEU register definition

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 10/19] raw/ifpga: add SEU error handler

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 245 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 1825143..f5e6119 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -27,6 +27,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -605,6 +607,236 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -652,6 +884,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -677,6 +910,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -696,6 +933,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -720,6 +958,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 11/19] raw/ifpga: add PCIe BDF devices tree scan

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 551 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 562 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index f5e6119..01ff76a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_malloc.h>
@@ -17,7 +19,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -25,6 +27,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -37,6 +40,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -64,6 +73,494 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(
+		struct ifpga_rawdev *ifpga_rdev, int force_disable);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+		int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
+					sensor->name, value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(
+	struct ifpga_rawdev *ifpga_rdev, int force_disable)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable || force_disable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev,
+					gsd_enable);
+				if (ret == 1 && !gsd_enable) {
+					gsd_enable = 1;
+					i = -1;
+				}
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -372,8 +869,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -842,6 +1340,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -855,7 +1354,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -869,6 +1368,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -987,6 +1494,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1018,13 +1526,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1038,7 +1565,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1065,6 +1593,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 12/19] net/ipn3ke: remove configuration for i40e port bonding

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/meson.build                            |   7 +-
 drivers/net/ipn3ke/Makefile                    |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c             | 289 +++----------------------
 drivers/net/ipn3ke/ipn3ke_representor.c        |   8 +-
 drivers/net/ipn3ke/meson.build                 |   2 +-
 drivers/raw/ifpga/meson.build                  |   6 +
 drivers/raw/ifpga/rte_rawdev_ifpga_version.map |   6 +
 7 files changed, 62 insertions(+), 258 deletions(-)

diff --git a/drivers/meson.build b/drivers/meson.build
index 2ed2e95..afdbb3b 100644
--- a/drivers/meson.build
+++ b/drivers/meson.build
@@ -9,12 +9,12 @@ endif
 dpdk_driver_classes = ['common',
 	       'bus',
 	       'mempool', # depends on common and bus.
+               'raw',
 	       'net',     # depends on common, bus and mempool.
 	       'crypto',  # depends on common, bus and mempool (net in future).
 	       'compress', # depends on common, bus, mempool.
 	       'event',   # depends on common, bus, mempool and net.
-	       'baseband', # depends on common and bus.
-	       'raw']     # depends on common, bus, mempool, net and event.
+	       'baseband']
 
 default_cflags = machine_args
 if cc.has_argument('-Wno-format-truncation')
@@ -157,6 +157,9 @@ foreach class:dpdk_driver_classes
 
 			set_variable('shared_@0@'.format(lib_name), shared_dep)
 			set_variable('static_@0@'.format(lib_name), static_dep)
+			dependency_name = ''.join(lib_name.split('rte_'))
+			message('drivers/@0@: Defining dependency "@1@"'.format(
+					drv_path, dependency_name))
 		endif # build
 	endforeach
 
diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..2c65e49 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index 28d8aaf..3051cdf 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -324,7 +325,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -342,6 +344,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -409,9 +417,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -462,7 +467,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -489,7 +498,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -501,6 +515,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -508,6 +538,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -553,254 +584,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index d37f5e2..7e5d29d 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2918,8 +2919,11 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		rte_pmd_i40e_set_switch_dev(rpst->i40e_pf_eth_port_id,
+					    rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
diff --git a/drivers/net/ipn3ke/meson.build b/drivers/net/ipn3ke/meson.build
index 74b4d7c..4ea57b9 100644
--- a/drivers/net/ipn3ke/meson.build
+++ b/drivers/net/ipn3ke/meson.build
@@ -14,4 +14,4 @@ sources += files('ipn3ke_ethdev.c',
 	'ipn3ke_representor.c',
 	'ipn3ke_tm.c',
 	'ipn3ke_flow.c')
-deps += ['bus_ifpga', 'sched']
+deps += ['bus_ifpga', 'sched', 'pmd_i40e', 'rawdev', 'rawdev_ifpga']
diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
index 0ab6fd7..dbd74b9 100644
--- a/drivers/raw/ifpga/meson.build
+++ b/drivers/raw/ifpga/meson.build
@@ -8,13 +8,19 @@ objs = [base_objs]
 
 dep = dependency('libfdt', required: false)
 if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
 	build = false
 	reason = 'missing dependency, "libfdt"'
 endif
 deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
 	'bus_vdev', 'bus_ifpga', 'net']
+ext_deps += dep
+
 sources = files('ifpga_rawdev.c')
 
 includes += include_directories('base')
+includes += include_directories('../../net/ipn3ke')
 
 allow_experimental_apis = true
diff --git a/drivers/raw/ifpga/rte_rawdev_ifpga_version.map b/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
index 9b9ab1a..acea4d7 100644
--- a/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
+++ b/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
@@ -2,3 +2,9 @@ DPDK_18.05 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	ifpga_rawdev_get;
+} DPDK_18.05;
\ No newline at end of file
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v11 12/19] net/ipn3ke: remove configuration for i40e port bonding

[Ye Xiaolong]

+Bruce for the meson change.

Thanks,
Xiaolong

On 10/21, Andy Pei wrote:
>From: Rosen Xu <rosen.xu@intel.com>
>
>The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
>so it doesn't need to provide configuration for i40e port bonding.
>
>Signed-off-by: Rosen Xu <rosen.xu@intel.com>
>Signed-off-by: Andy Pei <andy.pei@intel.com>
>---
> drivers/meson.build                            |   7 +-
> drivers/net/ipn3ke/Makefile                    |   2 +
> drivers/net/ipn3ke/ipn3ke_ethdev.c             | 289 +++----------------------
> drivers/net/ipn3ke/ipn3ke_representor.c        |   8 +-
> drivers/net/ipn3ke/meson.build                 |   2 +-
> drivers/raw/ifpga/meson.build                  |   6 +
> drivers/raw/ifpga/rte_rawdev_ifpga_version.map |   6 +
> 7 files changed, 62 insertions(+), 258 deletions(-)
>
>diff --git a/drivers/meson.build b/drivers/meson.build
>index 2ed2e95..afdbb3b 100644
>--- a/drivers/meson.build
>+++ b/drivers/meson.build
>@@ -9,12 +9,12 @@ endif
> dpdk_driver_classes = ['common',
> 	       'bus',
> 	       'mempool', # depends on common and bus.
>+               'raw',
> 	       'net',     # depends on common, bus and mempool.
> 	       'crypto',  # depends on common, bus and mempool (net in future).
> 	       'compress', # depends on common, bus, mempool.
> 	       'event',   # depends on common, bus, mempool and net.
>-	       'baseband', # depends on common and bus.
>-	       'raw']     # depends on common, bus, mempool, net and event.
>+	       'baseband']
> 
> default_cflags = machine_args
> if cc.has_argument('-Wno-format-truncation')
>@@ -157,6 +157,9 @@ foreach class:dpdk_driver_classes
> 
> 			set_variable('shared_@0@'.format(lib_name), shared_dep)
> 			set_variable('static_@0@'.format(lib_name), static_dep)
>+			dependency_name = ''.join(lib_name.split('rte_'))
>+			message('drivers/@0@: Defining dependency "@1@"'.format(
>+					drv_path, dependency_name))
> 		endif # build
> 	endforeach
> 
>diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
>index 8c3ae37..2c65e49 100644
>--- a/drivers/net/ipn3ke/Makefile
>+++ b/drivers/net/ipn3ke/Makefile
>@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
> CFLAGS += -O3
> CFLAGS += $(WERROR_FLAGS)
> CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
>+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
>+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
> LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
> LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
> LDLIBS += -lrte_bus_ifpga
>diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
>index 28d8aaf..3051cdf 100644
>--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
>+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
>@@ -19,6 +19,7 @@
> #include <rte_bus_ifpga.h>
> #include <ifpga_common.h>
> #include <ifpga_logs.h>
>+#include <ifpga_rawdev.h>
> 
> #include "ipn3ke_rawdev_api.h"
> #include "ipn3ke_flow.h"
>@@ -324,7 +325,8 @@
> 				"LineSideMACType", &mac_type);
> 	hw->retimer.mac_type = (int)mac_type;
> 
>-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
>+	hw->acc_tm = 0;
>+	hw->acc_flow = 0;
> 
> 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
> 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
>@@ -342,6 +344,12 @@
> 		/* After reset, wait until init done */
> 		if (ipn3ke_vbng_init_done(hw))
> 			return -1;
>+
>+		hw->acc_tm = 1;
>+		hw->acc_flow = 1;
>+
>+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
>+			IPN3KE_READ_REG(hw, 0));
> 	}
> 
> 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
>@@ -409,9 +417,6 @@
> 		hw->flow_hw_enable = 1;
> 	}
> 
>-	hw->acc_tm = 0;
>-	hw->acc_flow = 0;
>-
> 	return 0;
> }
> 
>@@ -462,7 +467,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
> {
> 	char name[RTE_ETH_NAME_MAX_LEN];
> 	struct ipn3ke_hw *hw;
>-	int i, retval;
>+	struct rte_eth_dev *i40e_eth;
>+	struct ifpga_rawdev *ifpga_dev;
>+	uint16_t port_id;
>+	int i, j, retval;
>+	char *fvl_bdf;
> 
> 	/* check if the AFU device has been probed already */
> 	/* allocate shared mcp_vswitch structure */
>@@ -489,7 +498,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
> 	if (retval)
> 		return retval;
> 
>+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
>+		if (!ifpga_dev)
>+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
>+
> 	/* probe representor ports */
>+	j = 0;
> 	for (i = 0; i < hw->port_num; i++) {
> 		struct ipn3ke_rpst rpst = {
> 			.port_id = i,
>@@ -501,6 +515,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
> 		snprintf(name, sizeof(name), "net_%s_representor_%d",
> 			afu_dev->device.name, i);
> 
>+		for (; j < 8; j++) {
>+			fvl_bdf = ifpga_dev->fvl_bdf[j];
>+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
>+				&port_id);
>+			if (retval) {
>+				continue;
>+			} else {
>+				i40e_eth = &rte_eth_devices[port_id];
>+				rpst.i40e_pf_eth = i40e_eth;
>+				rpst.i40e_pf_eth_port_id = port_id;
>+
>+				j++;
>+				break;
>+			}
>+		}
>+
> 		retval = rte_eth_dev_create(&afu_dev->device, name,
> 			sizeof(struct ipn3ke_rpst), NULL, NULL,
> 			ipn3ke_rpst_init, &rpst);
>@@ -508,6 +538,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
> 		if (retval)
> 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
> 								name);
>+
> 	}
> 
> 	return 0;
>@@ -553,254 +584,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
> 
> RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
> 
>-static const char * const valid_args[] = {
>-#define IPN3KE_AFU_NAME         "afu"
>-		IPN3KE_AFU_NAME,
>-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
>-		IPN3KE_FPGA_ACCELERATION_LIST,
>-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
>-		IPN3KE_I40E_PF_LIST,
>-		NULL
>-};
>-
>-static int
>-ipn3ke_cfg_parse_acc_list(const char *afu_name,
>-	const char *acc_list_name)
>-{
>-	struct rte_afu_device *afu_dev;
>-	struct ipn3ke_hw *hw;
>-	const char *p_source;
>-	char *p_start;
>-	char name[RTE_ETH_NAME_MAX_LEN];
>-
>-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
>-	if (!afu_dev)
>-		return -1;
>-	hw = afu_dev->shared.data;
>-	if (!hw)
>-		return -1;
>-
>-	p_source = acc_list_name;
>-	while (*p_source) {
>-		while ((*p_source == '{') || (*p_source == '|'))
>-			p_source++;
>-		p_start = name;
>-		while ((*p_source != '|') && (*p_source != '}'))
>-			*p_start++ = *p_source++;
>-		*p_start = 0;
>-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
>-			hw->acc_tm = 1;
>-
>-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
>-			hw->acc_flow = 1;
>-
>-		if (*p_source == '}')
>-			return 0;
>-	}
>-
>-	return 0;
>-}
>-
>-static int
>-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
>-	const char *pf_name)
>-{
>-	struct rte_eth_dev *i40e_eth, *rpst_eth;
>-	struct rte_afu_device *afu_dev;
>-	struct ipn3ke_rpst *rpst;
>-	struct ipn3ke_hw *hw;
>-	const char *p_source;
>-	char *p_start;
>-	char name[RTE_ETH_NAME_MAX_LEN];
>-	uint16_t port_id;
>-	int i;
>-	int ret = -1;
>-
>-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
>-	if (!afu_dev)
>-		return -1;
>-	hw = afu_dev->shared.data;
>-	if (!hw)
>-		return -1;
>-
>-	p_source = pf_name;
>-	for (i = 0; i < hw->port_num; i++) {
>-		snprintf(name, sizeof(name), "net_%s_representor_%d",
>-			afu_name, i);
>-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
>-		if (ret)
>-			return -1;
>-		rpst_eth = &rte_eth_devices[port_id];
>-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
>-
>-		while ((*p_source == '{') || (*p_source == '|'))
>-			p_source++;
>-		p_start = name;
>-		while ((*p_source != '|') && (*p_source != '}'))
>-			*p_start++ = *p_source++;
>-		*p_start = 0;
>-
>-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
>-		if (ret)
>-			return -1;
>-		i40e_eth = &rte_eth_devices[port_id];
>-
>-		rpst->i40e_pf_eth = i40e_eth;
>-		rpst->i40e_pf_eth_port_id = port_id;
>-
>-		if ((*p_source == '}') || !(*p_source))
>-			break;
>-	}
>-
>-	return 0;
>-}
>-
>-static int
>-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
>-{
>-	struct rte_devargs *devargs;
>-	struct rte_kvargs *kvlist = NULL;
>-	char *afu_name = NULL;
>-	char *acc_name = NULL;
>-	char *pf_name = NULL;
>-	int afu_name_en = 0;
>-	int acc_list_en = 0;
>-	int pf_list_en = 0;
>-	int ret = -1;
>-
>-	devargs = dev->device.devargs;
>-
>-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
>-	if (!kvlist) {
>-		IPN3KE_AFU_PMD_ERR("error when parsing param");
>-		goto end;
>-	}
>-
>-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
>-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
>-				       &rte_ifpga_get_string_arg,
>-				       &afu_name) < 0) {
>-			IPN3KE_AFU_PMD_ERR("error to parse %s",
>-				     IPN3KE_AFU_NAME);
>-			goto end;
>-		} else {
>-			afu_name_en = 1;
>-		}
>-	}
>-
>-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
>-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
>-				       &rte_ifpga_get_string_arg,
>-				       &acc_name) < 0) {
>-			IPN3KE_AFU_PMD_ERR("error to parse %s",
>-				     IPN3KE_FPGA_ACCELERATION_LIST);
>-			goto end;
>-		} else {
>-			acc_list_en = 1;
>-		}
>-	}
>-
>-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
>-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
>-				       &rte_ifpga_get_string_arg,
>-				       &pf_name) < 0) {
>-			IPN3KE_AFU_PMD_ERR("error to parse %s",
>-				     IPN3KE_I40E_PF_LIST);
>-			goto end;
>-		} else {
>-			pf_list_en = 1;
>-		}
>-	}
>-
>-	if (!afu_name_en) {
>-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
>-			  IPN3KE_AFU_NAME);
>-		goto end;
>-	}
>-
>-	if (!pf_list_en) {
>-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
>-			  IPN3KE_I40E_PF_LIST);
>-		goto end;
>-	}
>-
>-	if (acc_list_en) {
>-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
>-		if (ret) {
>-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
>-			  IPN3KE_FPGA_ACCELERATION_LIST);
>-			goto end;
>-		}
>-	} else {
>-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
>-			  IPN3KE_FPGA_ACCELERATION_LIST);
>-	}
>-
>-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
>-	if (ret)
>-		goto end;
>-end:
>-	if (kvlist)
>-		rte_kvargs_free(kvlist);
>-	if (afu_name)
>-		free(afu_name);
>-	if (acc_name)
>-		free(acc_name);
>-
>-	return ret;
>-}
>-
>-static int
>-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
>-{
>-	struct rte_devargs *devargs;
>-	struct rte_kvargs *kvlist = NULL;
>-	char *afu_name = NULL;
>-	struct rte_afu_device *afu_dev;
>-	int ret = -1;
>-
>-	devargs = dev->device.devargs;
>-
>-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
>-	if (!kvlist) {
>-		IPN3KE_AFU_PMD_ERR("error when parsing param");
>-		goto end;
>-	}
>-
>-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
>-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
>-				       &rte_ifpga_get_string_arg,
>-				       &afu_name) < 0) {
>-			IPN3KE_AFU_PMD_ERR("error to parse %s",
>-				     IPN3KE_AFU_NAME);
>-		} else {
>-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
>-			if (!afu_dev)
>-				goto end;
>-			ret = ipn3ke_vswitch_remove(afu_dev);
>-		}
>-	} else {
>-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
>-	}
>-
>-end:
>-	if (kvlist)
>-		rte_kvargs_free(kvlist);
>-
>-	return ret;
>-}
>-
>-static struct rte_vdev_driver ipn3ke_cfg_driver = {
>-	.probe = ipn3ke_cfg_probe,
>-	.remove = ipn3ke_cfg_remove,
>-};
>-
>-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
>-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
>-	"afu=<string> "
>-	"fpga_acc=<string>"
>-	"i40e_pf=<string>");
>-
> RTE_INIT(ipn3ke_afu_init_log)
> {
> 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
>diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
>index d37f5e2..7e5d29d 100644
>--- a/drivers/net/ipn3ke/ipn3ke_representor.c
>+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
>@@ -20,6 +20,7 @@
> #include <rte_rawdev_pmd.h>
> #include <rte_bus_ifpga.h>
> #include <ifpga_logs.h>
>+#include <rte_pmd_i40e.h>
> 
> #include "ipn3ke_rawdev_api.h"
> #include "ipn3ke_flow.h"
>@@ -2918,8 +2919,11 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
> 	rpst->switch_domain_id = representor_param->switch_domain_id;
> 	rpst->port_id = representor_param->port_id;
> 	rpst->hw = representor_param->hw;
>-	rpst->i40e_pf_eth = NULL;
>-	rpst->i40e_pf_eth_port_id = 0xFFFF;
>+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
>+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
>+	if (rpst->i40e_pf_eth)
>+		rte_pmd_i40e_set_switch_dev(rpst->i40e_pf_eth_port_id,
>+					    rpst->ethdev);
> 
> 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
> 	if (!ethdev->data->mac_addrs) {
>diff --git a/drivers/net/ipn3ke/meson.build b/drivers/net/ipn3ke/meson.build
>index 74b4d7c..4ea57b9 100644
>--- a/drivers/net/ipn3ke/meson.build
>+++ b/drivers/net/ipn3ke/meson.build
>@@ -14,4 +14,4 @@ sources += files('ipn3ke_ethdev.c',
> 	'ipn3ke_representor.c',
> 	'ipn3ke_tm.c',
> 	'ipn3ke_flow.c')
>-deps += ['bus_ifpga', 'sched']
>+deps += ['bus_ifpga', 'sched', 'pmd_i40e', 'rawdev', 'rawdev_ifpga']
>diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
>index 0ab6fd7..dbd74b9 100644
>--- a/drivers/raw/ifpga/meson.build
>+++ b/drivers/raw/ifpga/meson.build
>@@ -8,13 +8,19 @@ objs = [base_objs]
> 
> dep = dependency('libfdt', required: false)
> if not dep.found()
>+	dep = cc.find_library('libfdt', required: false)
>+endif
>+if not dep.found()
> 	build = false
> 	reason = 'missing dependency, "libfdt"'
> endif
> deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
> 	'bus_vdev', 'bus_ifpga', 'net']
>+ext_deps += dep
>+
> sources = files('ifpga_rawdev.c')
> 
> includes += include_directories('base')
>+includes += include_directories('../../net/ipn3ke')
> 
> allow_experimental_apis = true
>diff --git a/drivers/raw/ifpga/rte_rawdev_ifpga_version.map b/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
>index 9b9ab1a..acea4d7 100644
>--- a/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
>+++ b/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
>@@ -2,3 +2,9 @@ DPDK_18.05 {
> 
> 	local: *;
> };
>+
>+EXPERIMENTAL {
>+	global:
>+
>+	ifpga_rawdev_get;
>+} DPDK_18.05;
>\ No newline at end of file
>-- 
>1.8.3.1
>

Re: [dpdk-dev] [PATCH v11 12/19] net/ipn3ke: remove configuration for i40e port bonding

[Bruce Richardson]

On Mon, Oct 21, 2019 at 03:23:09PM +0800, Ye Xiaolong wrote:
> +Bruce for the meson change.
> 
> Thanks,
> Xiaolong
> 
> On 10/21, Andy Pei wrote:
> >From: Rosen Xu <rosen.xu@intel.com>
> >
> >The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
> >so it doesn't need to provide configuration for i40e port bonding.
> >
> >Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> >Signed-off-by: Andy Pei <andy.pei@intel.com>
> >---
> > drivers/meson.build                            |   7 +-
> > drivers/net/ipn3ke/Makefile                    |   2 +
> > drivers/net/ipn3ke/ipn3ke_ethdev.c             | 289 +++----------------------
> > drivers/net/ipn3ke/ipn3ke_representor.c        |   8 +-
> > drivers/net/ipn3ke/meson.build                 |   2 +-
> > drivers/raw/ifpga/meson.build                  |   6 +
> > drivers/raw/ifpga/rte_rawdev_ifpga_version.map |   6 +
> > 7 files changed, 62 insertions(+), 258 deletions(-)
> >
> >diff --git a/drivers/meson.build b/drivers/meson.build
> >index 2ed2e95..afdbb3b 100644
> >--- a/drivers/meson.build
> >+++ b/drivers/meson.build
> >@@ -9,12 +9,12 @@ endif
> > dpdk_driver_classes = ['common',
> > 	       'bus',
> > 	       'mempool', # depends on common and bus.
> >+               'raw',
> > 	       'net',     # depends on common, bus and mempool.
> > 	       'crypto',  # depends on common, bus and mempool (net in future).
> > 	       'compress', # depends on common, bus, mempool.
> > 	       'event',   # depends on common, bus, mempool and net.
> >-	       'baseband', # depends on common and bus.
> >-	       'raw']     # depends on common, bus, mempool, net and event.
> >+	       'baseband']
> > 

This needs an explanation in the commit log, and possibly should be a
separate patch. The comment on "net" need to be updated explaining why it
comes after raw.

> > default_cflags = machine_args
> > if cc.has_argument('-Wno-format-truncation')
> >@@ -157,6 +157,9 @@ foreach class:dpdk_driver_classes
> > 
> > 			set_variable('shared_@0@'.format(lib_name), shared_dep)
> > 			set_variable('static_@0@'.format(lib_name), static_dep)
> >+			dependency_name = ''.join(lib_name.split('rte_'))
> >+			message('drivers/@0@: Defining dependency "@1@"'.format(
> >+					drv_path, dependency_name))
> > 		endif # build
> > 	endforeach
> > 

This doesn't belong in this patch. It's covered by:
http://patches.dpdk.org/patch/59470/, so feel free to ack that patch
instead.

> >diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
> >index 8c3ae37..2c65e49 100644

<snip>

> >diff --git a/drivers/net/ipn3ke/meson.build b/drivers/net/ipn3ke/meson.build
> >index 74b4d7c..4ea57b9 100644
> >--- a/drivers/net/ipn3ke/meson.build
> >+++ b/drivers/net/ipn3ke/meson.build
> >@@ -14,4 +14,4 @@ sources += files('ipn3ke_ethdev.c',
> > 	'ipn3ke_representor.c',
> > 	'ipn3ke_tm.c',
> > 	'ipn3ke_flow.c')
> >-deps += ['bus_ifpga', 'sched']
> >+deps += ['bus_ifpga', 'sched', 'pmd_i40e', 'rawdev', 'rawdev_ifpga']

Minor nit: rawdev_ifpga already depends on rawdev and buf_ifpga, so you can
drop them to shorten the list if you like.
	deps += ['sched', 'pmd_i40e', 'rawdev_ifpga']

> >diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
> >index 0ab6fd7..dbd74b9 100644
> >--- a/drivers/raw/ifpga/meson.build
> >+++ b/drivers/raw/ifpga/meson.build
> >@@ -8,13 +8,19 @@ objs = [base_objs]
> > 
> > dep = dependency('libfdt', required: false)
> > if not dep.found()
> >+	dep = cc.find_library('libfdt', required: false)
> >+endif
> >+if not dep.found()
> > 	build = false
> > 	reason = 'missing dependency, "libfdt"'
> > endif
> > deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
> > 	'bus_vdev', 'bus_ifpga', 'net']

Similarly, if you want to cut this list down: 'kvargs' is always depended
upon by everything since its a dependency of eal, every rawdev already
depends upon the rawdev library and bus_pci depends upon pci.

	deps += ['bus_pci', 'bus_vdev', 'bus_ifpga', 'net']

should work on it's own.

Cutting the dependency list can help speed up configuration a little as fewer
dependency chains need to be iterated.

[dpdk-dev] [PATCH v11 13/19] raw/ifpga/base: add secure support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 137 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 ++++++++++++-----
 4 files changed, 198 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..748ab56 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,63 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager")) {
+			sysmgr_offset = offset;
+			break;
+		}
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +576,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 14/19] raw/ifpga/base: configure FEC mode

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 15/19] raw/ifpga/base: clean fme errors

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occurred.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 5d6d630..5905eac 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 01ff76a..95f079a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1174,6 +1174,25 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1183,6 +1202,9 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 16/19] raw/ifpga/base: add new API get board info

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..9f0147d 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disaggregate;
+	u8 lightweight;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..799d67d 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweight = id.lightweiht;
+	fme->board_info.disaggregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweight:%s\n"
+			"disaggregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweight),
+			check_support(fme->board_info.disaggregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 17/19] raw/ifpga: add lightweight fpga image support

[Andy Pei]

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 95f079a..e87af66 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -835,6 +835,8 @@ static int set_surprise_link_check_aer(
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -861,22 +863,40 @@ static int set_surprise_link_check_aer(
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweight) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
-		uuid.b + 8, sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 18/19] raw/ifpga/base: add multiple cards support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, there is one MAX10 chip in each card, and
all of the sensors are connected to MAX10 chip. To support multiple
cards in one server, we introducing a sensor device list under
intel_max10_device instead of a global list. On the other hand, we
using separate intel_max10_device instance for each opae_adatper.

Add mutex lock on do_transaction() function for SPI driver to avoid
race condition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c            |  40 +++++++---
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          |  14 ++--
 drivers/raw/ifpga/base/opae_hw_api.h          |  15 ++--
 drivers/raw/ifpga/base/opae_i2c.c             |  44 ++++++++---
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_intel_max10.c     | 110 ++++++++++++++------------
 drivers/raw/ifpga/base/opae_intel_max10.h     |  19 +++--
 drivers/raw/ifpga/base/opae_spi.c             |   5 --
 drivers/raw/ifpga/base/opae_spi.h             |   3 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  44 +++++++++--
 drivers/raw/ifpga/ifpga_rawdev.c              |  14 ++--
 12 files changed, 205 insertions(+), 109 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 799d67d..c31a94c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -839,8 +839,13 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	struct ifpga_hw *hw;
 	u32 val;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
@@ -854,7 +859,10 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	fme->board_info.seu = id.seu;
 	fme->board_info.ptp = id.ptp;
 
-	dev_info(fme, "found: board: %s type: %s\n",
+	dev_info(fme, "found: PCI dev: %02x:%02x:%x board: %s type: %s\n",
+			hw->pci_data->bus,
+			hw->pci_data->devid,
+			hw->pci_data->function,
 			board_major_to_string(fme->board_info.major),
 			board_type_to_string(fme->board_info.type));
 
@@ -882,11 +890,11 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
-	if (max10_sys_read(MAX10_BUILD_VER, &val))
+	if (max10_sys_read(fme->max10_dev, MAX10_BUILD_VER, &val))
 		return -EINVAL;
 	fme->board_info.max10_version = val & 0xffffff;
 
-	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+	if (max10_sys_read(fme->max10_dev, NIOS2_FW_VERSION, &val))
 		return -EINVAL;
 	fme->board_info.nios_fw_version = val & 0xffffff;
 
@@ -897,12 +905,12 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	return 0;
 }
 
-static int spi_self_checking(void)
+static int spi_self_checking(struct intel_max10_device *dev)
 {
 	u32 val;
 	int ret;
 
-	ret = max10_sys_read(MAX10_TEST_REG, &val);
+	ret = max10_sys_read(dev, MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
@@ -937,10 +945,11 @@ static int fme_spi_init(struct ifpga_feature *feature)
 		goto spi_fail;
 	}
 
+
 	fme->max10_dev = max10;
 
 	/* SPI self test */
-	if (spi_self_checking()) {
+	if (spi_self_checking(max10)) {
 		ret = -EIO;
 		goto max10_fail;
 	}
@@ -1041,8 +1050,18 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 	struct ifpga_fme_hw *fme = (struct ifpga_fme_hw *)feature->parent;
 	struct altera_spi_device *spi_master;
 	struct intel_max10_device *max10;
+	struct ifpga_hw *hw;
+	struct opae_manager *mgr;
 	int ret = 0;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
+	mgr = hw->adapter->mgr;
+	if (!mgr)
+		return -ENODEV;
+
 	dev_info(fme, "FME SPI Master (NIOS) Init.\n");
 	dev_debug(fme, "FME SPI base addr %p.\n",
 			feature->addr);
@@ -1080,12 +1099,15 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 		goto release_dev;
 	}
 
+	max10->bus = hw->pci_data->bus;
+
 	fme_get_board_interface(fme);
 
 	fme->max10_dev = max10;
+	mgr->sensor_list = &max10->opae_sensor_list;
 
 	/* SPI self test */
-	if (spi_self_checking())
+	if (spi_self_checking(max10))
 		goto spi_fail;
 
 	return ret;
@@ -1344,7 +1366,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(dev, PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1372,7 +1394,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(sensor->value_reg, value)) {
+	if (max10_sys_read(dev, sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_debug.c b/drivers/raw/ifpga/base/opae_debug.c
index 88f2d5c..dad3ea3 100644
--- a/drivers/raw/ifpga/base/opae_debug.c
+++ b/drivers/raw/ifpga/base/opae_debug.c
@@ -59,6 +59,9 @@ static void opae_adapter_data_dump(void *data)
 		opae_log("OPAE Adapter Type = PCI\n");
 		opae_log("PCI Device ID: 0x%04x\n", d_pci->device_id);
 		opae_log("PCI Vendor ID: 0x%04x\n", d_pci->vendor_id);
+		opae_log("PCI bus: 0x%04x\n", d_pci->bus);
+		opae_log("PCI devid: 0x%04x\n", d_pci->devid);
+		opae_log("PCI function: 0x%04x\n", d_pci->function);
 
 		for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 			r = &d_pci->region[i];
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 1ccc967..c969dfe 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -583,11 +583,12 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_id(unsigned int id)
+opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (sensor->id == id)
 			return sensor;
 
@@ -601,11 +602,12 @@ struct opae_sensor_info *
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_name(const char *name)
+opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (!strcmp(sensor->name, name))
 			return sensor;
 
@@ -630,7 +632,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_name(name);
+	sensor = opae_mgr_get_sensor_by_name(mgr, name);
 	if (!sensor)
 		return -ENODEV;
 
@@ -658,7 +660,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_id(id);
+	sensor = opae_mgr_get_sensor_by_id(mgr, id);
 	if (!sensor)
 		return -ENODEV;
 
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index b78fbd5..cdf369f 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -40,6 +40,7 @@ struct opae_manager {
 	struct opae_adapter *adapter;
 	struct opae_manager_ops *ops;
 	struct opae_manager_networking_ops *network_ops;
+	struct opae_sensor_list *sensor_list;
 	void *data;
 };
 
@@ -75,9 +76,8 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
-extern struct opae_sensor_list opae_sensor_list;
-#define opae_mgr_for_each_sensor(sensor) \
-	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+#define opae_mgr_for_each_sensor(mgr, sensor) \
+	TAILQ_FOREACH(sensor, mgr->sensor_list, node)
 
 /* OPAE Manager APIs */
 struct opae_manager *
@@ -88,8 +88,10 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
-struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
-struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id);
 int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
 		const char *name, unsigned int *value);
 int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
@@ -241,6 +243,9 @@ struct opae_adapter_data_pci {
 	enum opae_adapter_type type;
 	u16 device_id;
 	u16 vendor_id;
+	u16 bus; /*Device bus for PCI */
+	u16 devid; /* Device ID */
+	u16 function; /* Device function */
 	struct opae_reg_region region[PCI_MAX_RESOURCE];
 	int vfio_dev_fd;  /* VFIO device file descriptor */
 };
diff --git a/drivers/raw/ifpga/base/opae_i2c.c b/drivers/raw/ifpga/base/opae_i2c.c
index f8bc247..846d751 100644
--- a/drivers/raw/ifpga/base/opae_i2c.c
+++ b/drivers/raw/ifpga/base/opae_i2c.c
@@ -28,6 +28,9 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 {
 	u8 msgbuf[2];
 	int i = 0;
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
 
 	if (flags & I2C_FLAG_ADDR16)
 		msgbuf[i++] = offset >> 8;
@@ -49,10 +52,16 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		},
 	};
 
-	if (!dev->xfer)
-		return -ENODEV;
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	ret = i2c_transfer(dev, msg, 2);
 
-	return i2c_transfer(dev, msg, 2);
+exit:
+	pthread_mutex_unlock(&dev->lock);
+	return ret;
 }
 
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
@@ -63,12 +72,18 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	int ret;
 	int i = 0;
 
-	if (!dev->xfer)
-		return -ENODEV;
+	pthread_mutex_lock(&dev->lock);
+
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
 
 	buf = opae_malloc(I2C_MAX_OFFSET_LEN + len);
-	if (!buf)
-		return -ENOMEM;
+	if (!buf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
 
 	msg.addr = slave_addr;
 	msg.flags = 0;
@@ -84,6 +99,8 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	ret = i2c_transfer(dev, &msg, 1);
 
 	opae_free(buf);
+exit:
+	pthread_mutex_unlock(&dev->lock);
 	return ret;
 }
 
@@ -477,14 +494,19 @@ struct altera_i2c_dev *altera_i2c_probe(void *base)
 	dev->i2c_clk = dev->i2c_param.ref_clk * 1000000;
 	dev->xfer = altera_i2c_xfer;
 
+	if (pthread_mutex_init(&dev->lock, NULL))
+		return NULL;
+
 	altera_i2c_hardware_init(dev);
 
 	return dev;
 }
 
-int altera_i2c_remove(struct altera_i2c_dev *dev)
+void altera_i2c_remove(struct altera_i2c_dev *dev)
 {
-	altera_i2c_disable(dev);
-
-	return 0;
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
+		altera_i2c_disable(dev);
+		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/base/opae_i2c.h b/drivers/raw/ifpga/base/opae_i2c.h
index 8890c8f..266e127 100644
--- a/drivers/raw/ifpga/base/opae_i2c.h
+++ b/drivers/raw/ifpga/base/opae_i2c.h
@@ -93,6 +93,7 @@ struct altera_i2c_dev {
 	u32 isr_mask;
 	u8 *buf;
 	int (*xfer)(struct altera_i2c_dev *dev, struct i2c_msg *msg, int num);
+	pthread_mutex_t lock;
 };
 
 /**
@@ -114,7 +115,7 @@ enum i2c_msg_flags {
 };
 
 struct altera_i2c_dev *altera_i2c_probe(void *base);
-int altera_i2c_remove(struct altera_i2c_dev *dev);
+void altera_i2c_remove(struct altera_i2c_dev *dev);
 int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		u32 offset, u8 *buf, u32 count);
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 748ab56..8e23ca1 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -5,45 +5,50 @@
 #include "opae_intel_max10.h"
 #include <libfdt.h>
 
-static struct intel_max10_device *g_max10;
-
-struct opae_sensor_list opae_sensor_list =
-	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
-
-int max10_reg_read(unsigned int reg, unsigned int *val)
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_read(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x\n", __func__, dev->bus, reg);
+
+	return spi_transaction_read(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)val);
 }
 
-int max10_reg_write(unsigned int reg, unsigned int val)
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val)
 {
 	unsigned int tmp = val;
 
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_write(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x, val:0x%x\n", __func__,
+			dev->bus, reg, val);
+
+	return spi_transaction_write(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)&tmp);
 }
 
-int max10_sys_read(unsigned int offset, unsigned int *val)
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_read(g_max10->base + offset, val);
+
+	return max10_reg_read(dev, dev->base + offset, val);
 }
 
-int max10_sys_write(unsigned int offset, unsigned int val)
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_write(g_max10->base + offset, val);
+	return max10_reg_write(dev, dev->base + offset, val);
 }
 
 static struct max10_compatible_id max10_id_table[] = {
@@ -86,8 +91,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
-static int altera_nor_flash_read(u32 offset,
-		void *buffer, u32 len)
+static int altera_nor_flash_read(struct intel_max10_device *dev,
+	u32 offset, void *buffer, u32 len)
 {
 	int word_len;
 	int i;
@@ -95,13 +100,13 @@ static int altera_nor_flash_read(u32 offset,
 	unsigned int value;
 	int ret;
 
-	if (!buffer || len <= 0)
+	if (!dev || !buffer || len <= 0)
 		return -ENODEV;
 
 	word_len = len/4;
 
 	for (i = 0; i < word_len; i++) {
-		ret = max10_reg_read(offset + i*4,
+		ret = max10_reg_read(dev, offset + i*4,
 				&value);
 		if (ret)
 			return -EBUSY;
@@ -112,12 +117,12 @@ static int altera_nor_flash_read(u32 offset,
 	return 0;
 }
 
-static int enable_nor_flash(bool on)
+static int enable_nor_flash(struct intel_max10_device *dev, bool on)
 {
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_sys_read(RSU_REG, &val);
+	ret = max10_sys_read(dev, RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -128,7 +133,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_sys_write(RSU_REG, val);
+	return max10_sys_write(dev, RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -140,7 +145,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_sys_read(DT_AVAIL_REG, &val);
+	ret = max10_sys_read(max10, DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -151,19 +156,19 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
+	ret = max10_sys_read(max10, DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
 	}
 
-	ret = enable_nor_flash(true);
+	ret = enable_nor_flash(max10, true);
 	if (ret) {
 		dev_err(max10 "fail to enable flash\n");
 		return ret;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	ret = altera_nor_flash_read(max10, dt_addr, &hdr, sizeof(hdr));
 	if (ret) {
 		dev_err(max10 "read fdt header fail\n");
 		goto done;
@@ -188,7 +193,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		goto done;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	ret = altera_nor_flash_read(max10, dt_addr, fdt_root, dt_size);
 	if (ret) {
 		dev_err(max10 "cannot read device table\n");
 		goto done;
@@ -207,7 +212,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	max10->fdt_root = fdt_root;
 
 done:
-	ret = enable_nor_flash(false);
+	ret = enable_nor_flash(max10, false);
 
 	if (ret && fdt_root)
 		opae_free(fdt_root);
@@ -298,12 +303,12 @@ static int fdt_get_named_reg(const void *fdt, int node, const char *name,
 	return fdt_get_reg(fdt, node, idx, start, size);
 }
 
-static void max10_sensor_uinit(void)
+static void max10_sensor_uinit(struct intel_max10_device *dev)
 {
 	struct opae_sensor_info *info;
 
-	TAILQ_FOREACH(info, &opae_sensor_list, node) {
-		TAILQ_REMOVE(&opae_sensor_list, info, node);
+	TAILQ_FOREACH(info, &dev->opae_sensor_list, node) {
+		TAILQ_REMOVE(&dev->opae_sensor_list, info, node);
 		opae_free(info);
 	}
 }
@@ -313,8 +318,8 @@ static bool sensor_reg_valid(struct sensor_reg *reg)
 	return !!reg->size;
 }
 
-static int max10_add_sensor(struct raw_sensor_info *info,
-		struct opae_sensor_info *sensor)
+static int max10_add_sensor(struct intel_max10_device *dev,
+	struct raw_sensor_info *info, struct opae_sensor_info *sensor)
 {
 	int i;
 	int ret = 0;
@@ -332,12 +337,15 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_sys_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(dev, info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
-		if (val == 0xdeadbeef)
+		if (val == 0xdeadbeef) {
+			dev_debug(dev, "%s: sensor:%s invalid 0x%x at:%d\n",
+				__func__, sensor->name, val, i);
 			continue;
+		}
 
 		val *= info->multiplier;
 
@@ -458,7 +466,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
 		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
 
-		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+		dev_debug(dev, "found sensor from DTB: %s: %s: %u: %u\n",
 				raw->name, raw->type,
 				raw->id, raw->multiplier);
 
@@ -473,15 +481,16 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 			goto free_sensor;
 		}
 
-		if (max10_add_sensor(raw, sensor)) {
+		if (max10_add_sensor(dev, raw, sensor)) {
 			ret = -EINVAL;
 			opae_free(sensor);
 			goto free_sensor;
 		}
 
-		if (sensor->flags & OPAE_SENSOR_VALID)
-			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
-		else
+		if (sensor->flags & OPAE_SENSOR_VALID) {
+			TAILQ_INSERT_TAIL(&dev->opae_sensor_list, sensor, node);
+			dev_info(dev, "found valid sensor: %s\n", sensor->name);
+		} else
 			opae_free(sensor);
 
 		opae_free(raw);
@@ -492,7 +501,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 free_sensor:
 	if (raw)
 		opae_free(raw);
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 	return ret;
 }
 
@@ -500,7 +509,7 @@ static int check_max10_version(struct intel_max10_device *dev)
 {
 	unsigned int v;
 
-	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+	if (!max10_reg_read(dev, MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
 				&v)) {
 		if (v != 0xffffffff) {
 			dev_info(dev, "secure MAX10 detected\n");
@@ -565,6 +574,8 @@ struct intel_max10_device *
 	if (!dev)
 		return NULL;
 
+	TAILQ_INIT(&dev->opae_sensor_list);
+
 	dev->spi_master = spi;
 
 	dev->spi_tran_dev = spi_transaction_init(spi, chipselect);
@@ -573,9 +584,6 @@ struct intel_max10_device *
 		goto free_dev;
 	}
 
-	/* set the max10 device firstly */
-	g_max10 = dev;
-
 	/* check the max10 version */
 	ret = check_max10_version(dev);
 	if (ret) {
@@ -601,7 +609,7 @@ struct intel_max10_device *
 	}
 
 	/* read FPGA loading information */
-	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
+	ret = max10_sys_read(dev, FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
 		goto release_max10_hw;
@@ -611,14 +619,13 @@ struct intel_max10_device *
 	return dev;
 
 release_max10_hw:
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return NULL;
@@ -629,7 +636,7 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
@@ -637,7 +644,6 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index e632941..123cdc4 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -26,6 +26,9 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SECURE		BIT(6)
 #define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
@@ -33,6 +36,8 @@ struct intel_max10_device {
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
 	unsigned int base; /* max10 base address */
+	u16 bus;
+	struct opae_sensor_list opae_sensor_list;
 };
 
 /* retimer speed */
@@ -136,17 +141,19 @@ struct opae_retimer_status {
 
 #define DFT_MAX_SIZE		0x7e0000
 
-int max10_reg_read(unsigned int reg, unsigned int *val);
-int max10_reg_write(unsigned int reg, unsigned int val);
-int max10_sys_read(unsigned int offset, unsigned int *val);
-int max10_sys_write(unsigned int offset, unsigned int val);
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val);
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val);
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val);
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
-/** List of opae sensors */
-TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
 
 #define SENSOR_REG_VALUE 0x0
 #define SENSOR_REG_HIGH_WARN 0x1
diff --git a/drivers/raw/ifpga/base/opae_spi.c b/drivers/raw/ifpga/base/opae_spi.c
index cc52782..bfdc83e 100644
--- a/drivers/raw/ifpga/base/opae_spi.c
+++ b/drivers/raw/ifpga/base/opae_spi.c
@@ -181,7 +181,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 	u32 rxd;
 	unsigned int tx_data;
 	u32 status;
-	int retry = 0;
 	int ret;
 
 	while (count < dev->len) {
@@ -194,10 +193,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 				return -EIO;
 			if (status & ALTERA_SPI_STATUS_RRDY_MSK)
 				break;
-			if (retry++ > SPI_MAX_RETRY) {
-				dev_err(dev, "%s, read timeout\n", __func__);
-				return -EBUSY;
-			}
 		}
 
 		ret = spi_reg_read(dev, ALTERA_SPI_RXDATA, &rxd);
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index 6355deb..d20a4c3 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -38,7 +38,7 @@
 #define SPI_WRITE 0x20
 #define WRITE_DATA_MASK GENMASK_ULL(31, 0)
 
-#define SPI_MAX_RETRY 100000
+#define SPI_MAX_RETRY 1000000
 
 #define TYPE_SPI 0
 #define TYPE_NIOS_SPI 1
@@ -102,6 +102,7 @@ struct spi_transaction_dev {
 	struct altera_spi_device *dev;
 	int chipselect;
 	struct spi_tran_buffer *buffer;
+	pthread_mutex_t lock;
 };
 
 struct spi_tran_header {
diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 06ca625..013efee 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -154,6 +154,8 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	unsigned int rx_len = 0;
 	int retry = 0;
 	int spi_flags;
+	unsigned long timeout = msecs_to_timer_cycles(1000);
+	unsigned long ticks;
 	unsigned int resp_max_len = 2 * resp_len;
 
 	print_buffer("before bytes:", send_data, send_len);
@@ -207,8 +209,11 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	if (ret != SPI_FOUND_EOP) {
 		tx_buffer = NULL;
 		tx_len = 0;
-		if (retry++ > 10) {
-			dev_err(NULL, "cannot found valid data from SPI\n");
+		ticks = rte_get_timer_cycles();
+		if (time_after(ticks, timeout) &&
+				retry++ > SPI_MAX_RETRY) {
+			dev_err(NULL, "Have retry %d, found invalid packet data\n",
+				retry);
 			return -EBUSY;
 		}
 
@@ -427,23 +432,36 @@ static int do_transaction(struct spi_transaction_dev *dev, unsigned int addr,
 int spi_transaction_read(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_READ : SPI_TRAN_NON_SEQ_READ);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 int spi_transaction_write(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_WRITE : SPI_TRAN_NON_SEQ_WRITE);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 		int chipselect)
 {
 	struct spi_transaction_dev *spi_tran_dev;
+	int ret;
 
 	spi_tran_dev = opae_malloc(sizeof(struct spi_transaction_dev));
 	if (!spi_tran_dev)
@@ -453,18 +471,28 @@ struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 	spi_tran_dev->chipselect = chipselect;
 
 	spi_tran_dev->buffer = opae_malloc(sizeof(struct spi_tran_buffer));
-	if (!spi_tran_dev->buffer) {
-		opae_free(spi_tran_dev);
-		return NULL;
+	if (!spi_tran_dev->buffer)
+		goto err;
+
+	ret = pthread_mutex_init(&spi_tran_dev->lock, NULL);
+	if (ret) {
+		dev_err(spi_tran_dev, "fail to init mutex lock\n");
+		goto err;
 	}
 
 	return spi_tran_dev;
+
+err:
+	opae_free(spi_tran_dev);
+	return NULL;
 }
 
 void spi_transaction_remove(struct spi_transaction_dev *dev)
 {
 	if (dev && dev->buffer)
 		opae_free(dev->buffer);
-	if (dev)
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
 		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index e87af66..7253644 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -360,7 +360,7 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 	if (!mgr)
 		return -ENODEV;
 
-	opae_mgr_for_each_sensor(sensor) {
+	opae_mgr_for_each_sensor(mgr, sensor) {
 		if (!(sensor->flags & OPAE_SENSOR_VALID))
 			goto fail;
 
@@ -369,8 +369,8 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 			goto fail;
 
 		if (value == 0xdeadbeef) {
-			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
-					sensor->name, value);
+			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value %x\n",
+					raw_dev->dev_id, sensor->name, value);
 			continue;
 		}
 
@@ -393,8 +393,9 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 		/* monitor 12V AUX sensor */
 		if (!strcmp(sensor->name, "12V AUX Voltage")) {
 			if (value < AUX_VOLTAGE_WARN) {
-				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
-						sensor->name, value);
+				IFPGA_RAWDEV_PMD_INFO(
+					"%s reach theshold %d mV\n",
+					sensor->name, value);
 				*gsd_start = true;
 				break;
 			}
@@ -1433,6 +1434,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->bus = pci_dev->addr.bus;
+	data->devid = pci_dev->addr.devid;
+	data->function = pci_dev->addr.function;
 	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
-- 
1.8.3.1

[dpdk-dev] [PATCH v11 19/19] raw/ifpga: introducing new irq API

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing new register and unregister API for ifpga interrupt.
1. register FME and AFU interrupt
   ifpga_register_msix_irq()

2. unregister FME and AFU interrupt
   ifpga_unregister_msix_irq()

On PAC N3000 card, there is one PCIe MSIX interrupt for FME managerment,
like the error report, thermal management, we use this interrupt in
ifpga_rawdev device driver. on the other hand, there are about 4 PCIe
MSIX interrupts are reserved for AFU which end-user can use those
interrupts in their AFU logic design. End-user can use those APIs to
register interrupt handler in their AFU drivers.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 105 +++++++++++++++++++++++++++------------
 drivers/raw/ifpga/ifpga_rawdev.h |  14 ++++++
 2 files changed, 88 insertions(+), 31 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 7253644..753dbad 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -78,6 +78,10 @@
 static int ifpga_monitor_start;
 static pthread_t ifpga_monitor_start_thread;
 
+#define IFPGA_MAX_IRQ 12
+/* 0 for FME interrupt, others are reserved for AFU irq */
+static struct rte_intr_handle ifpga_irq_handle[IFPGA_MAX_IRQ];
+
 static struct ifpga_rawdev *
 ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
 static int set_surprise_link_check_aer(
@@ -1331,53 +1335,90 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 	fme_err_handle_catfatal_error(mgr);
 }
 
-static struct rte_intr_handle fme_intr_handle;
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg)
+{
+	struct rte_intr_handle intr_handle;
+
+	if (type == IFPGA_FME_IRQ)
+		intr_handle = ifpga_irq_handle[0];
+	else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+	rte_intr_efd_disable(&intr_handle);
+
+	return rte_intr_callback_unregister(&intr_handle,
+			handler, arg);
+}
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg)
 {
 	int ret;
-	struct fpga_fme_err_irq_set err_irq_set;
+	struct rte_intr_handle intr_handle;
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_accelerator *acc;
 
-	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+	adapter = ifpga_rawdev_get_priv(dev);
+	if (!adapter)
+		return -ENODEV;
 
-	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
-	if (ret)
-		return -EINVAL;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
 
-	fme_intr_handle.fd = fme_intr_handle.efds[0];
+	if (type == IFPGA_FME_IRQ) {
+		intr_handle = ifpga_irq_handle[0];
+		count = 1;
+	} else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
-			fme_intr_handle.vfio_dev_fd,
-			fme_intr_handle.efds[0], fme_intr_handle.fd);
+	intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
 
-	err_irq_set.evtfd = fme_intr_handle.efds[0];
-	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	ret = rte_intr_efd_enable(&intr_handle, count);
 	if (ret)
-		return -EINVAL;
+		return -ENODEV;
+
+	intr_handle.fd = intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("register %s irq, vfio_fd=%d, fd=%d\n",
+			name, intr_handle.vfio_dev_fd,
+			intr_handle.fd);
+
+	if (type == IFPGA_FME_IRQ) {
+		struct fpga_fme_err_irq_set err_irq_set;
+		err_irq_set.evtfd = intr_handle.efds[0];
+
+		ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+		if (ret)
+			return -EINVAL;
+	} else if (type == IFPGA_AFU_IRQ) {
+		acc = opae_adapter_get_acc(adapter, port_id);
+		if (!acc)
+			return -EINVAL;
 
-	/* register FME interrupt using DPDK API */
-	ret = rte_intr_callback_register(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
+		ret = opae_acc_set_irq(acc, vec_start, count, intr_handle.efds);
+		if (ret)
+			return -EINVAL;
+	}
+
+	/* register interrupt handler using DPDK API */
+	ret = rte_intr_callback_register(&intr_handle,
+			handler, (void *)arg);
 	if (ret)
 		return -EINVAL;
 
-	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+	IFPGA_RAWDEV_PMD_INFO("success register %s interrupt\n", name);
 
 	return 0;
 }
 
 static int
-ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
-{
-	rte_intr_efd_disable(&fme_intr_handle);
-
-	return rte_intr_callback_unregister(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
-}
-
-static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -1463,7 +1504,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
-	ret = ifpga_register_fme_interrupt(mgr);
+	ret = ifpga_register_msix_irq(rawdev, 0, IFPGA_FME_IRQ, 0, 0,
+			fme_interrupt_handler, "fme_irq", mgr);
 	if (ret)
 		goto free_adapter_data;
 
@@ -1515,7 +1557,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	if (!mgr)
 		return -ENODEV;
 
-	if (ifpga_unregister_fme_interrupt(mgr))
+	if (ifpga_unregister_msix_irq(IFPGA_FME_IRQ, 0,
+				fme_interrupt_handler, mgr))
 		return -EINVAL;
 
 	opae_adapter_data_free(adapter->data);
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index bd42083..7754beb 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -62,4 +62,18 @@ struct ifpga_rawdev {
 struct ifpga_rawdev *
 ifpga_rawdev_get(const struct rte_rawdev *rawdev);
 
+enum ifpga_irq_type {
+	IFPGA_FME_IRQ = 0,
+	IFPGA_AFU_IRQ = 1,
+};
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg);
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 02/19] raw/ifpga/base: add irq support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME global error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

To build this feature, CONFIG_RTE_EAL_VFIO should be enabled.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                         |  2 +-
 config/common_linux                        |  6 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 59 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 19 ++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 18 +++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 19 ++++++++++
 6 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/config/common_base b/config/common_base
index e843a21..68a4f70 100644
--- a/config/common_base
+++ b/config/common_base
@@ -772,7 +772,7 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
 #
 # Compile PMD for Intel FPGA raw device
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
 
 #
 # Compile PMD for Intel IOAT raw device
diff --git a/config/common_linux b/config/common_linux
index 96e2e1f..a78b8c6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -68,3 +68,9 @@ CONFIG_RTE_LIBRTE_HINIC_PMD=y
 # Hisilicon HNS3 PMD driver
 #
 CONFIG_RTE_LIBRTE_HNS3_PMD=y
+
+#
+# Compile PMD for Intel FPGA raw device
+# To compile, CONFIG_RTE_EAL_VFIO should be enabled.
+#
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..0f852a7 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,61 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+#define FPGA_MAX_MSIX_VEC_COUNT	128
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MAX_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	/* 0 < irq_set->count < FPGA_MAX_MSIX_VEC_COUNT */
+	irq_set->count = count ?
+		(count > FPGA_MAX_MSIX_VEC_COUNT ?
+		 FPGA_MAX_MSIX_VEC_COUNT : count) : 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..2978c79 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,28 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&fme->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..c0aaf01 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,27 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..189f762 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,28 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 03/19] raw/ifpga/base: clear pending bit

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 2978c79..be041ec 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 04/19] raw/ifpga/base: add SEU error support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 31 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index be041ec..5d6d630 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,33 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr(struct ifpga_fme_hw *fme,
+		u64 *val, bool high)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+				FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	if (high)
+		*val = readq(&fme_err->seu_emr_h);
+	else
+		*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +297,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr(fme, &prop->data, 0);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr(fme, &prop->data, 1);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 05/19] raw/ifpga/base: add device tree support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

To eliminate build error, we currently do not compile raw/ifpga
and net/ipn3ke. User should install libfdt and libfdt-devel first,
modify config/common_linux, CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
to CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y, modify config/common_base,
CONFIG_RTE_LIBRTE_IPN3KE_PMD=n to CONFIG_RTE_LIBRTE_IPN3KE_PMD=y.
Then this function can work.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                        |   2 +-
 config/common_linux                       |   2 +-
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 mk/rte.app.mk                             |   2 +-
 5 files changed, 196 insertions(+), 3 deletions(-)

diff --git a/config/common_base b/config/common_base
index 68a4f70..c0c9d5e 100644
--- a/config/common_base
+++ b/config/common_base
@@ -336,7 +336,7 @@ CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
 #
 # Compile burst-oriented IPN3KE PMD driver
 #
-CONFIG_RTE_LIBRTE_IPN3KE_PMD=y
+CONFIG_RTE_LIBRTE_IPN3KE_PMD=n
 
 #
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
diff --git a/config/common_linux b/config/common_linux
index a78b8c6..c5cf3d6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -73,4 +73,4 @@ CONFIG_RTE_LIBRTE_HNS3_PMD=y
 # Compile PMD for Intel FPGA raw device
 # To compile, CONFIG_RTE_EAL_VFIO should be enabled.
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index b91273f..28f3ef3 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -321,7 +321,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 06/19] raw/ifpga/base: align the send buffer for SPI

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 07/19] raw/ifpga/base: add sensor support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 08/19] raw/ifpga/base: introducing sensor APIs

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 09/19] raw/ifpga/base: update SEU register definition

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 10/19] raw/ifpga: add SEU error handler

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 245 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 1825143..f5e6119 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -27,6 +27,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -605,6 +607,236 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -652,6 +884,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -677,6 +910,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -696,6 +933,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -720,6 +958,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 11/19] raw/ifpga: add PCIe BDF devices tree scan

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 551 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 562 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index f5e6119..01ff76a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_malloc.h>
@@ -17,7 +19,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -25,6 +27,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -37,6 +40,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -64,6 +73,494 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(
+		struct ifpga_rawdev *ifpga_rdev, int force_disable);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+		int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
+					sensor->name, value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(
+	struct ifpga_rawdev *ifpga_rdev, int force_disable)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable || force_disable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev,
+					gsd_enable);
+				if (ret == 1 && !gsd_enable) {
+					gsd_enable = 1;
+					i = -1;
+				}
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -372,8 +869,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -842,6 +1340,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -855,7 +1354,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -869,6 +1368,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -987,6 +1494,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1018,13 +1526,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1038,7 +1565,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1065,6 +1593,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 12/19] net/ipn3ke: remove configuration for i40e port bonding

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/meson.build                            |   7 +-
 drivers/net/ipn3ke/Makefile                    |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c             | 289 +++----------------------
 drivers/net/ipn3ke/ipn3ke_representor.c        |   8 +-
 drivers/net/ipn3ke/meson.build                 |   2 +-
 drivers/raw/ifpga/meson.build                  |   6 +
 drivers/raw/ifpga/rte_rawdev_ifpga_version.map |   3 +
 7 files changed, 59 insertions(+), 258 deletions(-)

diff --git a/drivers/meson.build b/drivers/meson.build
index 2ed2e95..afdbb3b 100644
--- a/drivers/meson.build
+++ b/drivers/meson.build
@@ -9,12 +9,12 @@ endif
 dpdk_driver_classes = ['common',
 	       'bus',
 	       'mempool', # depends on common and bus.
+               'raw',
 	       'net',     # depends on common, bus and mempool.
 	       'crypto',  # depends on common, bus and mempool (net in future).
 	       'compress', # depends on common, bus, mempool.
 	       'event',   # depends on common, bus, mempool and net.
-	       'baseband', # depends on common and bus.
-	       'raw']     # depends on common, bus, mempool, net and event.
+	       'baseband']
 
 default_cflags = machine_args
 if cc.has_argument('-Wno-format-truncation')
@@ -157,6 +157,9 @@ foreach class:dpdk_driver_classes
 
 			set_variable('shared_@0@'.format(lib_name), shared_dep)
 			set_variable('static_@0@'.format(lib_name), static_dep)
+			dependency_name = ''.join(lib_name.split('rte_'))
+			message('drivers/@0@: Defining dependency "@1@"'.format(
+					drv_path, dependency_name))
 		endif # build
 	endforeach
 
diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..2c65e49 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index 28d8aaf..3051cdf 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -324,7 +325,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -342,6 +344,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -409,9 +417,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -462,7 +467,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -489,7 +498,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -501,6 +515,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -508,6 +538,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -553,254 +584,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index d37f5e2..7e5d29d 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2918,8 +2919,11 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		rte_pmd_i40e_set_switch_dev(rpst->i40e_pf_eth_port_id,
+					    rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
diff --git a/drivers/net/ipn3ke/meson.build b/drivers/net/ipn3ke/meson.build
index 74b4d7c..4ea57b9 100644
--- a/drivers/net/ipn3ke/meson.build
+++ b/drivers/net/ipn3ke/meson.build
@@ -14,4 +14,4 @@ sources += files('ipn3ke_ethdev.c',
 	'ipn3ke_representor.c',
 	'ipn3ke_tm.c',
 	'ipn3ke_flow.c')
-deps += ['bus_ifpga', 'sched']
+deps += ['bus_ifpga', 'sched', 'pmd_i40e', 'rawdev', 'rawdev_ifpga']
diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
index 0ab6fd7..dbd74b9 100644
--- a/drivers/raw/ifpga/meson.build
+++ b/drivers/raw/ifpga/meson.build
@@ -8,13 +8,19 @@ objs = [base_objs]
 
 dep = dependency('libfdt', required: false)
 if not dep.found()
+	dep = cc.find_library('libfdt', required: false)
+endif
+if not dep.found()
 	build = false
 	reason = 'missing dependency, "libfdt"'
 endif
 deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
 	'bus_vdev', 'bus_ifpga', 'net']
+ext_deps += dep
+
 sources = files('ifpga_rawdev.c')
 
 includes += include_directories('base')
+includes += include_directories('../../net/ipn3ke')
 
 allow_experimental_apis = true
diff --git a/drivers/raw/ifpga/rte_rawdev_ifpga_version.map b/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
index 9b9ab1a..a6b50aa 100644
--- a/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
+++ b/drivers/raw/ifpga/rte_rawdev_ifpga_version.map
@@ -1,4 +1,7 @@
 DPDK_18.05 {
+	global:
+
+	ifpga_rawdev_get;
 
 	local: *;
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 13/19] raw/ifpga/base: add secure support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 137 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 ++++++++++++-----
 4 files changed, 198 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..748ab56 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,63 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager")) {
+			sysmgr_offset = offset;
+			break;
+		}
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +576,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 14/19] raw/ifpga/base: configure FEC mode

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 15/19] raw/ifpga/base: clean fme errors

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occurred.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 5d6d630..5905eac 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 01ff76a..95f079a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1174,6 +1174,25 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1183,6 +1202,9 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 16/19] raw/ifpga/base: add new API get board info

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..9f0147d 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disaggregate;
+	u8 lightweight;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..799d67d 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweight = id.lightweiht;
+	fme->board_info.disaggregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweight:%s\n"
+			"disaggregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweight),
+			check_support(fme->board_info.disaggregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 17/19] raw/ifpga: add lightweight fpga image support

[Andy Pei]

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 95f079a..e87af66 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -835,6 +835,8 @@ static int set_surprise_link_check_aer(
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -861,22 +863,40 @@ static int set_surprise_link_check_aer(
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweight) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
-		uuid.b + 8, sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 18/19] raw/ifpga/base: add multiple cards support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, there is one MAX10 chip in each card, and
all of the sensors are connected to MAX10 chip. To support multiple
cards in one server, we introducing a sensor device list under
intel_max10_device instead of a global list. On the other hand, we
using separate intel_max10_device instance for each opae_adatper.

Add mutex lock on do_transaction() function for SPI driver to avoid
race condition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c            |  40 +++++++---
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          |  14 ++--
 drivers/raw/ifpga/base/opae_hw_api.h          |  15 ++--
 drivers/raw/ifpga/base/opae_i2c.c             |  44 ++++++++---
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_intel_max10.c     | 110 ++++++++++++++------------
 drivers/raw/ifpga/base/opae_intel_max10.h     |  19 +++--
 drivers/raw/ifpga/base/opae_spi.c             |   5 --
 drivers/raw/ifpga/base/opae_spi.h             |   3 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  44 +++++++++--
 drivers/raw/ifpga/ifpga_rawdev.c              |  14 ++--
 12 files changed, 205 insertions(+), 109 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 799d67d..c31a94c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -839,8 +839,13 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	struct ifpga_hw *hw;
 	u32 val;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
@@ -854,7 +859,10 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	fme->board_info.seu = id.seu;
 	fme->board_info.ptp = id.ptp;
 
-	dev_info(fme, "found: board: %s type: %s\n",
+	dev_info(fme, "found: PCI dev: %02x:%02x:%x board: %s type: %s\n",
+			hw->pci_data->bus,
+			hw->pci_data->devid,
+			hw->pci_data->function,
 			board_major_to_string(fme->board_info.major),
 			board_type_to_string(fme->board_info.type));
 
@@ -882,11 +890,11 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
-	if (max10_sys_read(MAX10_BUILD_VER, &val))
+	if (max10_sys_read(fme->max10_dev, MAX10_BUILD_VER, &val))
 		return -EINVAL;
 	fme->board_info.max10_version = val & 0xffffff;
 
-	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+	if (max10_sys_read(fme->max10_dev, NIOS2_FW_VERSION, &val))
 		return -EINVAL;
 	fme->board_info.nios_fw_version = val & 0xffffff;
 
@@ -897,12 +905,12 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	return 0;
 }
 
-static int spi_self_checking(void)
+static int spi_self_checking(struct intel_max10_device *dev)
 {
 	u32 val;
 	int ret;
 
-	ret = max10_sys_read(MAX10_TEST_REG, &val);
+	ret = max10_sys_read(dev, MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
@@ -937,10 +945,11 @@ static int fme_spi_init(struct ifpga_feature *feature)
 		goto spi_fail;
 	}
 
+
 	fme->max10_dev = max10;
 
 	/* SPI self test */
-	if (spi_self_checking()) {
+	if (spi_self_checking(max10)) {
 		ret = -EIO;
 		goto max10_fail;
 	}
@@ -1041,8 +1050,18 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 	struct ifpga_fme_hw *fme = (struct ifpga_fme_hw *)feature->parent;
 	struct altera_spi_device *spi_master;
 	struct intel_max10_device *max10;
+	struct ifpga_hw *hw;
+	struct opae_manager *mgr;
 	int ret = 0;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
+	mgr = hw->adapter->mgr;
+	if (!mgr)
+		return -ENODEV;
+
 	dev_info(fme, "FME SPI Master (NIOS) Init.\n");
 	dev_debug(fme, "FME SPI base addr %p.\n",
 			feature->addr);
@@ -1080,12 +1099,15 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 		goto release_dev;
 	}
 
+	max10->bus = hw->pci_data->bus;
+
 	fme_get_board_interface(fme);
 
 	fme->max10_dev = max10;
+	mgr->sensor_list = &max10->opae_sensor_list;
 
 	/* SPI self test */
-	if (spi_self_checking())
+	if (spi_self_checking(max10))
 		goto spi_fail;
 
 	return ret;
@@ -1344,7 +1366,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(dev, PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1372,7 +1394,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(sensor->value_reg, value)) {
+	if (max10_sys_read(dev, sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_debug.c b/drivers/raw/ifpga/base/opae_debug.c
index 88f2d5c..dad3ea3 100644
--- a/drivers/raw/ifpga/base/opae_debug.c
+++ b/drivers/raw/ifpga/base/opae_debug.c
@@ -59,6 +59,9 @@ static void opae_adapter_data_dump(void *data)
 		opae_log("OPAE Adapter Type = PCI\n");
 		opae_log("PCI Device ID: 0x%04x\n", d_pci->device_id);
 		opae_log("PCI Vendor ID: 0x%04x\n", d_pci->vendor_id);
+		opae_log("PCI bus: 0x%04x\n", d_pci->bus);
+		opae_log("PCI devid: 0x%04x\n", d_pci->devid);
+		opae_log("PCI function: 0x%04x\n", d_pci->function);
 
 		for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 			r = &d_pci->region[i];
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 1ccc967..c969dfe 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -583,11 +583,12 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_id(unsigned int id)
+opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (sensor->id == id)
 			return sensor;
 
@@ -601,11 +602,12 @@ struct opae_sensor_info *
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_name(const char *name)
+opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (!strcmp(sensor->name, name))
 			return sensor;
 
@@ -630,7 +632,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_name(name);
+	sensor = opae_mgr_get_sensor_by_name(mgr, name);
 	if (!sensor)
 		return -ENODEV;
 
@@ -658,7 +660,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_id(id);
+	sensor = opae_mgr_get_sensor_by_id(mgr, id);
 	if (!sensor)
 		return -ENODEV;
 
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index b78fbd5..cdf369f 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -40,6 +40,7 @@ struct opae_manager {
 	struct opae_adapter *adapter;
 	struct opae_manager_ops *ops;
 	struct opae_manager_networking_ops *network_ops;
+	struct opae_sensor_list *sensor_list;
 	void *data;
 };
 
@@ -75,9 +76,8 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
-extern struct opae_sensor_list opae_sensor_list;
-#define opae_mgr_for_each_sensor(sensor) \
-	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+#define opae_mgr_for_each_sensor(mgr, sensor) \
+	TAILQ_FOREACH(sensor, mgr->sensor_list, node)
 
 /* OPAE Manager APIs */
 struct opae_manager *
@@ -88,8 +88,10 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
-struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
-struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id);
 int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
 		const char *name, unsigned int *value);
 int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
@@ -241,6 +243,9 @@ struct opae_adapter_data_pci {
 	enum opae_adapter_type type;
 	u16 device_id;
 	u16 vendor_id;
+	u16 bus; /*Device bus for PCI */
+	u16 devid; /* Device ID */
+	u16 function; /* Device function */
 	struct opae_reg_region region[PCI_MAX_RESOURCE];
 	int vfio_dev_fd;  /* VFIO device file descriptor */
 };
diff --git a/drivers/raw/ifpga/base/opae_i2c.c b/drivers/raw/ifpga/base/opae_i2c.c
index f8bc247..846d751 100644
--- a/drivers/raw/ifpga/base/opae_i2c.c
+++ b/drivers/raw/ifpga/base/opae_i2c.c
@@ -28,6 +28,9 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 {
 	u8 msgbuf[2];
 	int i = 0;
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
 
 	if (flags & I2C_FLAG_ADDR16)
 		msgbuf[i++] = offset >> 8;
@@ -49,10 +52,16 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		},
 	};
 
-	if (!dev->xfer)
-		return -ENODEV;
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	ret = i2c_transfer(dev, msg, 2);
 
-	return i2c_transfer(dev, msg, 2);
+exit:
+	pthread_mutex_unlock(&dev->lock);
+	return ret;
 }
 
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
@@ -63,12 +72,18 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	int ret;
 	int i = 0;
 
-	if (!dev->xfer)
-		return -ENODEV;
+	pthread_mutex_lock(&dev->lock);
+
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
 
 	buf = opae_malloc(I2C_MAX_OFFSET_LEN + len);
-	if (!buf)
-		return -ENOMEM;
+	if (!buf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
 
 	msg.addr = slave_addr;
 	msg.flags = 0;
@@ -84,6 +99,8 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	ret = i2c_transfer(dev, &msg, 1);
 
 	opae_free(buf);
+exit:
+	pthread_mutex_unlock(&dev->lock);
 	return ret;
 }
 
@@ -477,14 +494,19 @@ struct altera_i2c_dev *altera_i2c_probe(void *base)
 	dev->i2c_clk = dev->i2c_param.ref_clk * 1000000;
 	dev->xfer = altera_i2c_xfer;
 
+	if (pthread_mutex_init(&dev->lock, NULL))
+		return NULL;
+
 	altera_i2c_hardware_init(dev);
 
 	return dev;
 }
 
-int altera_i2c_remove(struct altera_i2c_dev *dev)
+void altera_i2c_remove(struct altera_i2c_dev *dev)
 {
-	altera_i2c_disable(dev);
-
-	return 0;
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
+		altera_i2c_disable(dev);
+		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/base/opae_i2c.h b/drivers/raw/ifpga/base/opae_i2c.h
index 8890c8f..266e127 100644
--- a/drivers/raw/ifpga/base/opae_i2c.h
+++ b/drivers/raw/ifpga/base/opae_i2c.h
@@ -93,6 +93,7 @@ struct altera_i2c_dev {
 	u32 isr_mask;
 	u8 *buf;
 	int (*xfer)(struct altera_i2c_dev *dev, struct i2c_msg *msg, int num);
+	pthread_mutex_t lock;
 };
 
 /**
@@ -114,7 +115,7 @@ enum i2c_msg_flags {
 };
 
 struct altera_i2c_dev *altera_i2c_probe(void *base);
-int altera_i2c_remove(struct altera_i2c_dev *dev);
+void altera_i2c_remove(struct altera_i2c_dev *dev);
 int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		u32 offset, u8 *buf, u32 count);
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 748ab56..8e23ca1 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -5,45 +5,50 @@
 #include "opae_intel_max10.h"
 #include <libfdt.h>
 
-static struct intel_max10_device *g_max10;
-
-struct opae_sensor_list opae_sensor_list =
-	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
-
-int max10_reg_read(unsigned int reg, unsigned int *val)
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_read(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x\n", __func__, dev->bus, reg);
+
+	return spi_transaction_read(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)val);
 }
 
-int max10_reg_write(unsigned int reg, unsigned int val)
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val)
 {
 	unsigned int tmp = val;
 
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_write(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x, val:0x%x\n", __func__,
+			dev->bus, reg, val);
+
+	return spi_transaction_write(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)&tmp);
 }
 
-int max10_sys_read(unsigned int offset, unsigned int *val)
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_read(g_max10->base + offset, val);
+
+	return max10_reg_read(dev, dev->base + offset, val);
 }
 
-int max10_sys_write(unsigned int offset, unsigned int val)
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_write(g_max10->base + offset, val);
+	return max10_reg_write(dev, dev->base + offset, val);
 }
 
 static struct max10_compatible_id max10_id_table[] = {
@@ -86,8 +91,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
-static int altera_nor_flash_read(u32 offset,
-		void *buffer, u32 len)
+static int altera_nor_flash_read(struct intel_max10_device *dev,
+	u32 offset, void *buffer, u32 len)
 {
 	int word_len;
 	int i;
@@ -95,13 +100,13 @@ static int altera_nor_flash_read(u32 offset,
 	unsigned int value;
 	int ret;
 
-	if (!buffer || len <= 0)
+	if (!dev || !buffer || len <= 0)
 		return -ENODEV;
 
 	word_len = len/4;
 
 	for (i = 0; i < word_len; i++) {
-		ret = max10_reg_read(offset + i*4,
+		ret = max10_reg_read(dev, offset + i*4,
 				&value);
 		if (ret)
 			return -EBUSY;
@@ -112,12 +117,12 @@ static int altera_nor_flash_read(u32 offset,
 	return 0;
 }
 
-static int enable_nor_flash(bool on)
+static int enable_nor_flash(struct intel_max10_device *dev, bool on)
 {
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_sys_read(RSU_REG, &val);
+	ret = max10_sys_read(dev, RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -128,7 +133,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_sys_write(RSU_REG, val);
+	return max10_sys_write(dev, RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -140,7 +145,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_sys_read(DT_AVAIL_REG, &val);
+	ret = max10_sys_read(max10, DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -151,19 +156,19 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
+	ret = max10_sys_read(max10, DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
 	}
 
-	ret = enable_nor_flash(true);
+	ret = enable_nor_flash(max10, true);
 	if (ret) {
 		dev_err(max10 "fail to enable flash\n");
 		return ret;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	ret = altera_nor_flash_read(max10, dt_addr, &hdr, sizeof(hdr));
 	if (ret) {
 		dev_err(max10 "read fdt header fail\n");
 		goto done;
@@ -188,7 +193,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		goto done;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	ret = altera_nor_flash_read(max10, dt_addr, fdt_root, dt_size);
 	if (ret) {
 		dev_err(max10 "cannot read device table\n");
 		goto done;
@@ -207,7 +212,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	max10->fdt_root = fdt_root;
 
 done:
-	ret = enable_nor_flash(false);
+	ret = enable_nor_flash(max10, false);
 
 	if (ret && fdt_root)
 		opae_free(fdt_root);
@@ -298,12 +303,12 @@ static int fdt_get_named_reg(const void *fdt, int node, const char *name,
 	return fdt_get_reg(fdt, node, idx, start, size);
 }
 
-static void max10_sensor_uinit(void)
+static void max10_sensor_uinit(struct intel_max10_device *dev)
 {
 	struct opae_sensor_info *info;
 
-	TAILQ_FOREACH(info, &opae_sensor_list, node) {
-		TAILQ_REMOVE(&opae_sensor_list, info, node);
+	TAILQ_FOREACH(info, &dev->opae_sensor_list, node) {
+		TAILQ_REMOVE(&dev->opae_sensor_list, info, node);
 		opae_free(info);
 	}
 }
@@ -313,8 +318,8 @@ static bool sensor_reg_valid(struct sensor_reg *reg)
 	return !!reg->size;
 }
 
-static int max10_add_sensor(struct raw_sensor_info *info,
-		struct opae_sensor_info *sensor)
+static int max10_add_sensor(struct intel_max10_device *dev,
+	struct raw_sensor_info *info, struct opae_sensor_info *sensor)
 {
 	int i;
 	int ret = 0;
@@ -332,12 +337,15 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_sys_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(dev, info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
-		if (val == 0xdeadbeef)
+		if (val == 0xdeadbeef) {
+			dev_debug(dev, "%s: sensor:%s invalid 0x%x at:%d\n",
+				__func__, sensor->name, val, i);
 			continue;
+		}
 
 		val *= info->multiplier;
 
@@ -458,7 +466,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
 		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
 
-		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+		dev_debug(dev, "found sensor from DTB: %s: %s: %u: %u\n",
 				raw->name, raw->type,
 				raw->id, raw->multiplier);
 
@@ -473,15 +481,16 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 			goto free_sensor;
 		}
 
-		if (max10_add_sensor(raw, sensor)) {
+		if (max10_add_sensor(dev, raw, sensor)) {
 			ret = -EINVAL;
 			opae_free(sensor);
 			goto free_sensor;
 		}
 
-		if (sensor->flags & OPAE_SENSOR_VALID)
-			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
-		else
+		if (sensor->flags & OPAE_SENSOR_VALID) {
+			TAILQ_INSERT_TAIL(&dev->opae_sensor_list, sensor, node);
+			dev_info(dev, "found valid sensor: %s\n", sensor->name);
+		} else
 			opae_free(sensor);
 
 		opae_free(raw);
@@ -492,7 +501,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 free_sensor:
 	if (raw)
 		opae_free(raw);
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 	return ret;
 }
 
@@ -500,7 +509,7 @@ static int check_max10_version(struct intel_max10_device *dev)
 {
 	unsigned int v;
 
-	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+	if (!max10_reg_read(dev, MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
 				&v)) {
 		if (v != 0xffffffff) {
 			dev_info(dev, "secure MAX10 detected\n");
@@ -565,6 +574,8 @@ struct intel_max10_device *
 	if (!dev)
 		return NULL;
 
+	TAILQ_INIT(&dev->opae_sensor_list);
+
 	dev->spi_master = spi;
 
 	dev->spi_tran_dev = spi_transaction_init(spi, chipselect);
@@ -573,9 +584,6 @@ struct intel_max10_device *
 		goto free_dev;
 	}
 
-	/* set the max10 device firstly */
-	g_max10 = dev;
-
 	/* check the max10 version */
 	ret = check_max10_version(dev);
 	if (ret) {
@@ -601,7 +609,7 @@ struct intel_max10_device *
 	}
 
 	/* read FPGA loading information */
-	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
+	ret = max10_sys_read(dev, FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
 		goto release_max10_hw;
@@ -611,14 +619,13 @@ struct intel_max10_device *
 	return dev;
 
 release_max10_hw:
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return NULL;
@@ -629,7 +636,7 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
@@ -637,7 +644,6 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index e632941..123cdc4 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -26,6 +26,9 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SECURE		BIT(6)
 #define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
@@ -33,6 +36,8 @@ struct intel_max10_device {
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
 	unsigned int base; /* max10 base address */
+	u16 bus;
+	struct opae_sensor_list opae_sensor_list;
 };
 
 /* retimer speed */
@@ -136,17 +141,19 @@ struct opae_retimer_status {
 
 #define DFT_MAX_SIZE		0x7e0000
 
-int max10_reg_read(unsigned int reg, unsigned int *val);
-int max10_reg_write(unsigned int reg, unsigned int val);
-int max10_sys_read(unsigned int offset, unsigned int *val);
-int max10_sys_write(unsigned int offset, unsigned int val);
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val);
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val);
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val);
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
-/** List of opae sensors */
-TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
 
 #define SENSOR_REG_VALUE 0x0
 #define SENSOR_REG_HIGH_WARN 0x1
diff --git a/drivers/raw/ifpga/base/opae_spi.c b/drivers/raw/ifpga/base/opae_spi.c
index cc52782..bfdc83e 100644
--- a/drivers/raw/ifpga/base/opae_spi.c
+++ b/drivers/raw/ifpga/base/opae_spi.c
@@ -181,7 +181,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 	u32 rxd;
 	unsigned int tx_data;
 	u32 status;
-	int retry = 0;
 	int ret;
 
 	while (count < dev->len) {
@@ -194,10 +193,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 				return -EIO;
 			if (status & ALTERA_SPI_STATUS_RRDY_MSK)
 				break;
-			if (retry++ > SPI_MAX_RETRY) {
-				dev_err(dev, "%s, read timeout\n", __func__);
-				return -EBUSY;
-			}
 		}
 
 		ret = spi_reg_read(dev, ALTERA_SPI_RXDATA, &rxd);
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index 6355deb..d20a4c3 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -38,7 +38,7 @@
 #define SPI_WRITE 0x20
 #define WRITE_DATA_MASK GENMASK_ULL(31, 0)
 
-#define SPI_MAX_RETRY 100000
+#define SPI_MAX_RETRY 1000000
 
 #define TYPE_SPI 0
 #define TYPE_NIOS_SPI 1
@@ -102,6 +102,7 @@ struct spi_transaction_dev {
 	struct altera_spi_device *dev;
 	int chipselect;
 	struct spi_tran_buffer *buffer;
+	pthread_mutex_t lock;
 };
 
 struct spi_tran_header {
diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 06ca625..013efee 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -154,6 +154,8 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	unsigned int rx_len = 0;
 	int retry = 0;
 	int spi_flags;
+	unsigned long timeout = msecs_to_timer_cycles(1000);
+	unsigned long ticks;
 	unsigned int resp_max_len = 2 * resp_len;
 
 	print_buffer("before bytes:", send_data, send_len);
@@ -207,8 +209,11 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	if (ret != SPI_FOUND_EOP) {
 		tx_buffer = NULL;
 		tx_len = 0;
-		if (retry++ > 10) {
-			dev_err(NULL, "cannot found valid data from SPI\n");
+		ticks = rte_get_timer_cycles();
+		if (time_after(ticks, timeout) &&
+				retry++ > SPI_MAX_RETRY) {
+			dev_err(NULL, "Have retry %d, found invalid packet data\n",
+				retry);
 			return -EBUSY;
 		}
 
@@ -427,23 +432,36 @@ static int do_transaction(struct spi_transaction_dev *dev, unsigned int addr,
 int spi_transaction_read(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_READ : SPI_TRAN_NON_SEQ_READ);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 int spi_transaction_write(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_WRITE : SPI_TRAN_NON_SEQ_WRITE);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 		int chipselect)
 {
 	struct spi_transaction_dev *spi_tran_dev;
+	int ret;
 
 	spi_tran_dev = opae_malloc(sizeof(struct spi_transaction_dev));
 	if (!spi_tran_dev)
@@ -453,18 +471,28 @@ struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 	spi_tran_dev->chipselect = chipselect;
 
 	spi_tran_dev->buffer = opae_malloc(sizeof(struct spi_tran_buffer));
-	if (!spi_tran_dev->buffer) {
-		opae_free(spi_tran_dev);
-		return NULL;
+	if (!spi_tran_dev->buffer)
+		goto err;
+
+	ret = pthread_mutex_init(&spi_tran_dev->lock, NULL);
+	if (ret) {
+		dev_err(spi_tran_dev, "fail to init mutex lock\n");
+		goto err;
 	}
 
 	return spi_tran_dev;
+
+err:
+	opae_free(spi_tran_dev);
+	return NULL;
 }
 
 void spi_transaction_remove(struct spi_transaction_dev *dev)
 {
 	if (dev && dev->buffer)
 		opae_free(dev->buffer);
-	if (dev)
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
 		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index e87af66..7253644 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -360,7 +360,7 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 	if (!mgr)
 		return -ENODEV;
 
-	opae_mgr_for_each_sensor(sensor) {
+	opae_mgr_for_each_sensor(mgr, sensor) {
 		if (!(sensor->flags & OPAE_SENSOR_VALID))
 			goto fail;
 
@@ -369,8 +369,8 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 			goto fail;
 
 		if (value == 0xdeadbeef) {
-			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
-					sensor->name, value);
+			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value %x\n",
+					raw_dev->dev_id, sensor->name, value);
 			continue;
 		}
 
@@ -393,8 +393,9 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 		/* monitor 12V AUX sensor */
 		if (!strcmp(sensor->name, "12V AUX Voltage")) {
 			if (value < AUX_VOLTAGE_WARN) {
-				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
-						sensor->name, value);
+				IFPGA_RAWDEV_PMD_INFO(
+					"%s reach theshold %d mV\n",
+					sensor->name, value);
 				*gsd_start = true;
 				break;
 			}
@@ -1433,6 +1434,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->bus = pci_dev->addr.bus;
+	data->devid = pci_dev->addr.devid;
+	data->function = pci_dev->addr.function;
 	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
-- 
1.8.3.1

[dpdk-dev] [PATCH v10 19/19] raw/ifpga: introducing new irq API

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing new register and unregister API for ifpga interrupt.
1. register FME and AFU interrupt
   ifpga_register_msix_irq()

2. unregister FME and AFU interrupt
   ifpga_unregister_msix_irq()

On PAC N3000 card, there is one PCIe MSIX interrupt for FME managerment,
like the error report, thermal management, we use this interrupt in
ifpga_rawdev device driver. on the other hand, there are about 4 PCIe
MSIX interrupts are reserved for AFU which end-user can use those
interrupts in their AFU logic design. End-user can use those APIs to
register interrupt handler in their AFU drivers.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 105 +++++++++++++++++++++++++++------------
 drivers/raw/ifpga/ifpga_rawdev.h |  14 ++++++
 2 files changed, 88 insertions(+), 31 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 7253644..753dbad 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -78,6 +78,10 @@
 static int ifpga_monitor_start;
 static pthread_t ifpga_monitor_start_thread;
 
+#define IFPGA_MAX_IRQ 12
+/* 0 for FME interrupt, others are reserved for AFU irq */
+static struct rte_intr_handle ifpga_irq_handle[IFPGA_MAX_IRQ];
+
 static struct ifpga_rawdev *
 ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
 static int set_surprise_link_check_aer(
@@ -1331,53 +1335,90 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 	fme_err_handle_catfatal_error(mgr);
 }
 
-static struct rte_intr_handle fme_intr_handle;
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg)
+{
+	struct rte_intr_handle intr_handle;
+
+	if (type == IFPGA_FME_IRQ)
+		intr_handle = ifpga_irq_handle[0];
+	else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+	rte_intr_efd_disable(&intr_handle);
+
+	return rte_intr_callback_unregister(&intr_handle,
+			handler, arg);
+}
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg)
 {
 	int ret;
-	struct fpga_fme_err_irq_set err_irq_set;
+	struct rte_intr_handle intr_handle;
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_accelerator *acc;
 
-	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+	adapter = ifpga_rawdev_get_priv(dev);
+	if (!adapter)
+		return -ENODEV;
 
-	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
-	if (ret)
-		return -EINVAL;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
 
-	fme_intr_handle.fd = fme_intr_handle.efds[0];
+	if (type == IFPGA_FME_IRQ) {
+		intr_handle = ifpga_irq_handle[0];
+		count = 1;
+	} else if (type == IFPGA_AFU_IRQ)
+		intr_handle = ifpga_irq_handle[vec_start + 1];
 
-	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
-			fme_intr_handle.vfio_dev_fd,
-			fme_intr_handle.efds[0], fme_intr_handle.fd);
+	intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
 
-	err_irq_set.evtfd = fme_intr_handle.efds[0];
-	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	ret = rte_intr_efd_enable(&intr_handle, count);
 	if (ret)
-		return -EINVAL;
+		return -ENODEV;
+
+	intr_handle.fd = intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("register %s irq, vfio_fd=%d, fd=%d\n",
+			name, intr_handle.vfio_dev_fd,
+			intr_handle.fd);
+
+	if (type == IFPGA_FME_IRQ) {
+		struct fpga_fme_err_irq_set err_irq_set;
+		err_irq_set.evtfd = intr_handle.efds[0];
+
+		ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+		if (ret)
+			return -EINVAL;
+	} else if (type == IFPGA_AFU_IRQ) {
+		acc = opae_adapter_get_acc(adapter, port_id);
+		if (!acc)
+			return -EINVAL;
 
-	/* register FME interrupt using DPDK API */
-	ret = rte_intr_callback_register(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
+		ret = opae_acc_set_irq(acc, vec_start, count, intr_handle.efds);
+		if (ret)
+			return -EINVAL;
+	}
+
+	/* register interrupt handler using DPDK API */
+	ret = rte_intr_callback_register(&intr_handle,
+			handler, (void *)arg);
 	if (ret)
 		return -EINVAL;
 
-	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+	IFPGA_RAWDEV_PMD_INFO("success register %s interrupt\n", name);
 
 	return 0;
 }
 
 static int
-ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
-{
-	rte_intr_efd_disable(&fme_intr_handle);
-
-	return rte_intr_callback_unregister(&fme_intr_handle,
-			fme_interrupt_handler,
-			(void *)mgr);
-}
-
-static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -1463,7 +1504,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
-	ret = ifpga_register_fme_interrupt(mgr);
+	ret = ifpga_register_msix_irq(rawdev, 0, IFPGA_FME_IRQ, 0, 0,
+			fme_interrupt_handler, "fme_irq", mgr);
 	if (ret)
 		goto free_adapter_data;
 
@@ -1515,7 +1557,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	if (!mgr)
 		return -ENODEV;
 
-	if (ifpga_unregister_fme_interrupt(mgr))
+	if (ifpga_unregister_msix_irq(IFPGA_FME_IRQ, 0,
+				fme_interrupt_handler, mgr))
 		return -EINVAL;
 
 	opae_adapter_data_free(adapter->data);
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index bd42083..7754beb 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -62,4 +62,18 @@ struct ifpga_rawdev {
 struct ifpga_rawdev *
 ifpga_rawdev_get(const struct rte_rawdev *rawdev);
 
+enum ifpga_irq_type {
+	IFPGA_FME_IRQ = 0,
+	IFPGA_AFU_IRQ = 1,
+};
+
+int
+ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+		enum ifpga_irq_type type, int vec_start, int count,
+		rte_intr_callback_fn handler, const char *name,
+		void *arg);
+int
+ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+		int vec_start, rte_intr_callback_fn handler, void *arg);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 02/18] raw/ifpga/base: add irq support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME global error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

To build this feature, CONFIG_RTE_EAL_VFIO should be enabled.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                         |  2 +-
 config/common_linux                        |  6 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 59 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 19 ++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 18 +++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 19 ++++++++++
 6 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/config/common_base b/config/common_base
index e843a21..68a4f70 100644
--- a/config/common_base
+++ b/config/common_base
@@ -772,7 +772,7 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
 #
 # Compile PMD for Intel FPGA raw device
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
 
 #
 # Compile PMD for Intel IOAT raw device
diff --git a/config/common_linux b/config/common_linux
index 96e2e1f..a78b8c6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -68,3 +68,9 @@ CONFIG_RTE_LIBRTE_HINIC_PMD=y
 # Hisilicon HNS3 PMD driver
 #
 CONFIG_RTE_LIBRTE_HNS3_PMD=y
+
+#
+# Compile PMD for Intel FPGA raw device
+# To compile, CONFIG_RTE_EAL_VFIO should be enabled.
+#
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..0f852a7 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,61 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+#define FPGA_MAX_MSIX_VEC_COUNT	128
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MAX_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	/* 0 < irq_set->count < FPGA_MAX_MSIX_VEC_COUNT */
+	irq_set->count = count ?
+		(count > FPGA_MAX_MSIX_VEC_COUNT ?
+		 FPGA_MAX_MSIX_VEC_COUNT : count) : 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..2978c79 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,28 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&fme->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..c0aaf01 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,27 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..189f762 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,28 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 03/18] raw/ifpga/base: clear pending bit

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 2978c79..be041ec 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 04/18] raw/ifpga/base: add SEU error support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 31 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index be041ec..5d6d630 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,33 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr(struct ifpga_fme_hw *fme,
+		u64 *val, bool high)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+				FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	if (high)
+		*val = readq(&fme_err->seu_emr_h);
+	else
+		*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +297,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr(fme, &prop->data, 0);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr(fme, &prop->data, 1);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 05/18] raw/ifpga/base: add device tree support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

To eliminate build error, we currently do not compile raw/ifpga
and net/ipn3ke. User should install libfdt and libfdt-devel first,
modify config/common_linux, CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
to CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y, modify config/common_base,
CONFIG_RTE_LIBRTE_IPN3KE_PMD=n to CONFIG_RTE_LIBRTE_IPN3KE_PMD=y.
Then this function can work.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                        |   2 +-
 config/common_linux                       |   2 +-
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 mk/rte.app.mk                             |   2 +-
 5 files changed, 196 insertions(+), 3 deletions(-)

diff --git a/config/common_base b/config/common_base
index 68a4f70..c0c9d5e 100644
--- a/config/common_base
+++ b/config/common_base
@@ -336,7 +336,7 @@ CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
 #
 # Compile burst-oriented IPN3KE PMD driver
 #
-CONFIG_RTE_LIBRTE_IPN3KE_PMD=y
+CONFIG_RTE_LIBRTE_IPN3KE_PMD=n
 
 #
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
diff --git a/config/common_linux b/config/common_linux
index a78b8c6..c5cf3d6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -73,4 +73,4 @@ CONFIG_RTE_LIBRTE_HNS3_PMD=y
 # Compile PMD for Intel FPGA raw device
 # To compile, CONFIG_RTE_EAL_VFIO should be enabled.
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index b91273f..28f3ef3 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -321,7 +321,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 06/18] raw/ifpga/base: align the send buffer for SPI

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 07/18] raw/ifpga/base: add sensor support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 08/18] raw/ifpga/base: introducing sensor APIs

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 09/18] raw/ifpga/base: update SEU register definition

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 10/18] raw/ifpga: add SEU error handler

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 245 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 1825143..f5e6119 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -27,6 +27,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -605,6 +607,236 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -652,6 +884,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -677,6 +910,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -696,6 +933,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -720,6 +958,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 11/18] raw/ifpga: add PCIe BDF devices tree scan

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 551 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 562 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index f5e6119..01ff76a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_malloc.h>
@@ -17,7 +19,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -25,6 +27,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -37,6 +40,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -64,6 +73,494 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(
+		struct ifpga_rawdev *ifpga_rdev, int force_disable);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+		int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
+					sensor->name, value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(
+	struct ifpga_rawdev *ifpga_rdev, int force_disable)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable || force_disable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev,
+					gsd_enable);
+				if (ret == 1 && !gsd_enable) {
+					gsd_enable = 1;
+					i = -1;
+				}
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -372,8 +869,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -842,6 +1340,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -855,7 +1354,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -869,6 +1368,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -987,6 +1494,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1018,13 +1526,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1038,7 +1565,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1065,6 +1593,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 12/18] net/ipn3ke: remove configuration for i40e port bonding

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/ipn3ke/Makefile             |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c      | 289 ++++----------------------------
 drivers/net/ipn3ke/ipn3ke_representor.c |   8 +-
 3 files changed, 44 insertions(+), 255 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..2c65e49 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index 28d8aaf..3051cdf 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -324,7 +325,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -342,6 +344,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -409,9 +417,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -462,7 +467,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -489,7 +498,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -501,6 +515,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -508,6 +538,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -553,254 +584,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index d37f5e2..7e5d29d 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2918,8 +2919,11 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		rte_pmd_i40e_set_switch_dev(rpst->i40e_pf_eth_port_id,
+					    rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v9 12/18] net/ipn3ke: remove configuration for i40e port bonding

[Aaron Conole]

Andy Pei <andy.pei@intel.com> writes:

> From: Rosen Xu <rosen.xu@intel.com>
>
> The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
> so it doesn't need to provide configuration for i40e port bonding.
>
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/net/ipn3ke/Makefile             |   2 +

You'll also need to include a change to the meson file to ensure
the include path will work for meson based builds.

>  drivers/net/ipn3ke/ipn3ke_ethdev.c      | 289 ++++----------------------------
>  drivers/net/ipn3ke/ipn3ke_representor.c |   8 +-
>  3 files changed, 44 insertions(+), 255 deletions(-)
>
> diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
> index 8c3ae37..2c65e49 100644
> --- a/drivers/net/ipn3ke/Makefile
> +++ b/drivers/net/ipn3ke/Makefile
> @@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
>  CFLAGS += -O3
>  CFLAGS += $(WERROR_FLAGS)
>  CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
> +CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
> +CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
>  LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
>  LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
>  LDLIBS += -lrte_bus_ifpga
> diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
> index 28d8aaf..3051cdf 100644
> --- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
> +++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
> @@ -19,6 +19,7 @@
>  #include <rte_bus_ifpga.h>
>  #include <ifpga_common.h>
>  #include <ifpga_logs.h>
> +#include <ifpga_rawdev.h>
>  
>  #include "ipn3ke_rawdev_api.h"
>  #include "ipn3ke_flow.h"
> @@ -324,7 +325,8 @@
>  				"LineSideMACType", &mac_type);
>  	hw->retimer.mac_type = (int)mac_type;
>  
> -	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
> +	hw->acc_tm = 0;
> +	hw->acc_flow = 0;
>  
>  	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
>  		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
> @@ -342,6 +344,12 @@
>  		/* After reset, wait until init done */
>  		if (ipn3ke_vbng_init_done(hw))
>  			return -1;
> +
> +		hw->acc_tm = 1;
> +		hw->acc_flow = 1;
> +
> +		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
> +			IPN3KE_READ_REG(hw, 0));
>  	}
>  
>  	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
> @@ -409,9 +417,6 @@
>  		hw->flow_hw_enable = 1;
>  	}
>  
> -	hw->acc_tm = 0;
> -	hw->acc_flow = 0;
> -
>  	return 0;
>  }
>  
> @@ -462,7 +467,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
>  {
>  	char name[RTE_ETH_NAME_MAX_LEN];
>  	struct ipn3ke_hw *hw;
> -	int i, retval;
> +	struct rte_eth_dev *i40e_eth;
> +	struct ifpga_rawdev *ifpga_dev;
> +	uint16_t port_id;
> +	int i, j, retval;
> +	char *fvl_bdf;
>  
>  	/* check if the AFU device has been probed already */
>  	/* allocate shared mcp_vswitch structure */
> @@ -489,7 +498,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
>  	if (retval)
>  		return retval;
>  
> +	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
> +		if (!ifpga_dev)
> +			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
> +
>  	/* probe representor ports */
> +	j = 0;
>  	for (i = 0; i < hw->port_num; i++) {
>  		struct ipn3ke_rpst rpst = {
>  			.port_id = i,
> @@ -501,6 +515,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
>  		snprintf(name, sizeof(name), "net_%s_representor_%d",
>  			afu_dev->device.name, i);
>  
> +		for (; j < 8; j++) {
> +			fvl_bdf = ifpga_dev->fvl_bdf[j];
> +			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
> +				&port_id);
> +			if (retval) {
> +				continue;
> +			} else {
> +				i40e_eth = &rte_eth_devices[port_id];
> +				rpst.i40e_pf_eth = i40e_eth;
> +				rpst.i40e_pf_eth_port_id = port_id;
> +
> +				j++;
> +				break;
> +			}
> +		}
> +
>  		retval = rte_eth_dev_create(&afu_dev->device, name,
>  			sizeof(struct ipn3ke_rpst), NULL, NULL,
>  			ipn3ke_rpst_init, &rpst);
> @@ -508,6 +538,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
>  		if (retval)
>  			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
>  								name);
> +
>  	}
>  
>  	return 0;
> @@ -553,254 +584,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
>  
>  RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
>  
> -static const char * const valid_args[] = {
> -#define IPN3KE_AFU_NAME         "afu"
> -		IPN3KE_AFU_NAME,
> -#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
> -		IPN3KE_FPGA_ACCELERATION_LIST,
> -#define IPN3KE_I40E_PF_LIST     "i40e_pf"
> -		IPN3KE_I40E_PF_LIST,
> -		NULL
> -};
> -
> -static int
> -ipn3ke_cfg_parse_acc_list(const char *afu_name,
> -	const char *acc_list_name)
> -{
> -	struct rte_afu_device *afu_dev;
> -	struct ipn3ke_hw *hw;
> -	const char *p_source;
> -	char *p_start;
> -	char name[RTE_ETH_NAME_MAX_LEN];
> -
> -	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
> -	if (!afu_dev)
> -		return -1;
> -	hw = afu_dev->shared.data;
> -	if (!hw)
> -		return -1;
> -
> -	p_source = acc_list_name;
> -	while (*p_source) {
> -		while ((*p_source == '{') || (*p_source == '|'))
> -			p_source++;
> -		p_start = name;
> -		while ((*p_source != '|') && (*p_source != '}'))
> -			*p_start++ = *p_source++;
> -		*p_start = 0;
> -		if (!strcmp(name, "tm") && hw->tm_hw_enable)
> -			hw->acc_tm = 1;
> -
> -		if (!strcmp(name, "flow") && hw->flow_hw_enable)
> -			hw->acc_flow = 1;
> -
> -		if (*p_source == '}')
> -			return 0;
> -	}
> -
> -	return 0;
> -}
> -
> -static int
> -ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
> -	const char *pf_name)
> -{
> -	struct rte_eth_dev *i40e_eth, *rpst_eth;
> -	struct rte_afu_device *afu_dev;
> -	struct ipn3ke_rpst *rpst;
> -	struct ipn3ke_hw *hw;
> -	const char *p_source;
> -	char *p_start;
> -	char name[RTE_ETH_NAME_MAX_LEN];
> -	uint16_t port_id;
> -	int i;
> -	int ret = -1;
> -
> -	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
> -	if (!afu_dev)
> -		return -1;
> -	hw = afu_dev->shared.data;
> -	if (!hw)
> -		return -1;
> -
> -	p_source = pf_name;
> -	for (i = 0; i < hw->port_num; i++) {
> -		snprintf(name, sizeof(name), "net_%s_representor_%d",
> -			afu_name, i);
> -		ret = rte_eth_dev_get_port_by_name(name, &port_id);
> -		if (ret)
> -			return -1;
> -		rpst_eth = &rte_eth_devices[port_id];
> -		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
> -
> -		while ((*p_source == '{') || (*p_source == '|'))
> -			p_source++;
> -		p_start = name;
> -		while ((*p_source != '|') && (*p_source != '}'))
> -			*p_start++ = *p_source++;
> -		*p_start = 0;
> -
> -		ret = rte_eth_dev_get_port_by_name(name, &port_id);
> -		if (ret)
> -			return -1;
> -		i40e_eth = &rte_eth_devices[port_id];
> -
> -		rpst->i40e_pf_eth = i40e_eth;
> -		rpst->i40e_pf_eth_port_id = port_id;
> -
> -		if ((*p_source == '}') || !(*p_source))
> -			break;
> -	}
> -
> -	return 0;
> -}
> -
> -static int
> -ipn3ke_cfg_probe(struct rte_vdev_device *dev)
> -{
> -	struct rte_devargs *devargs;
> -	struct rte_kvargs *kvlist = NULL;
> -	char *afu_name = NULL;
> -	char *acc_name = NULL;
> -	char *pf_name = NULL;
> -	int afu_name_en = 0;
> -	int acc_list_en = 0;
> -	int pf_list_en = 0;
> -	int ret = -1;
> -
> -	devargs = dev->device.devargs;
> -
> -	kvlist = rte_kvargs_parse(devargs->args, valid_args);
> -	if (!kvlist) {
> -		IPN3KE_AFU_PMD_ERR("error when parsing param");
> -		goto end;
> -	}
> -
> -	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
> -		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
> -				       &rte_ifpga_get_string_arg,
> -				       &afu_name) < 0) {
> -			IPN3KE_AFU_PMD_ERR("error to parse %s",
> -				     IPN3KE_AFU_NAME);
> -			goto end;
> -		} else {
> -			afu_name_en = 1;
> -		}
> -	}
> -
> -	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
> -		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
> -				       &rte_ifpga_get_string_arg,
> -				       &acc_name) < 0) {
> -			IPN3KE_AFU_PMD_ERR("error to parse %s",
> -				     IPN3KE_FPGA_ACCELERATION_LIST);
> -			goto end;
> -		} else {
> -			acc_list_en = 1;
> -		}
> -	}
> -
> -	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
> -		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
> -				       &rte_ifpga_get_string_arg,
> -				       &pf_name) < 0) {
> -			IPN3KE_AFU_PMD_ERR("error to parse %s",
> -				     IPN3KE_I40E_PF_LIST);
> -			goto end;
> -		} else {
> -			pf_list_en = 1;
> -		}
> -	}
> -
> -	if (!afu_name_en) {
> -		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
> -			  IPN3KE_AFU_NAME);
> -		goto end;
> -	}
> -
> -	if (!pf_list_en) {
> -		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
> -			  IPN3KE_I40E_PF_LIST);
> -		goto end;
> -	}
> -
> -	if (acc_list_en) {
> -		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
> -		if (ret) {
> -			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
> -			  IPN3KE_FPGA_ACCELERATION_LIST);
> -			goto end;
> -		}
> -	} else {
> -		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
> -			  IPN3KE_FPGA_ACCELERATION_LIST);
> -	}
> -
> -	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
> -	if (ret)
> -		goto end;
> -end:
> -	if (kvlist)
> -		rte_kvargs_free(kvlist);
> -	if (afu_name)
> -		free(afu_name);
> -	if (acc_name)
> -		free(acc_name);
> -
> -	return ret;
> -}
> -
> -static int
> -ipn3ke_cfg_remove(struct rte_vdev_device *dev)
> -{
> -	struct rte_devargs *devargs;
> -	struct rte_kvargs *kvlist = NULL;
> -	char *afu_name = NULL;
> -	struct rte_afu_device *afu_dev;
> -	int ret = -1;
> -
> -	devargs = dev->device.devargs;
> -
> -	kvlist = rte_kvargs_parse(devargs->args, valid_args);
> -	if (!kvlist) {
> -		IPN3KE_AFU_PMD_ERR("error when parsing param");
> -		goto end;
> -	}
> -
> -	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
> -		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
> -				       &rte_ifpga_get_string_arg,
> -				       &afu_name) < 0) {
> -			IPN3KE_AFU_PMD_ERR("error to parse %s",
> -				     IPN3KE_AFU_NAME);
> -		} else {
> -			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
> -			if (!afu_dev)
> -				goto end;
> -			ret = ipn3ke_vswitch_remove(afu_dev);
> -		}
> -	} else {
> -		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
> -	}
> -
> -end:
> -	if (kvlist)
> -		rte_kvargs_free(kvlist);
> -
> -	return ret;
> -}
> -
> -static struct rte_vdev_driver ipn3ke_cfg_driver = {
> -	.probe = ipn3ke_cfg_probe,
> -	.remove = ipn3ke_cfg_remove,
> -};
> -
> -RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
> -RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
> -	"afu=<string> "
> -	"fpga_acc=<string>"
> -	"i40e_pf=<string>");
> -
>  RTE_INIT(ipn3ke_afu_init_log)
>  {
>  	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
> diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
> index d37f5e2..7e5d29d 100644
> --- a/drivers/net/ipn3ke/ipn3ke_representor.c
> +++ b/drivers/net/ipn3ke/ipn3ke_representor.c
> @@ -20,6 +20,7 @@
>  #include <rte_rawdev_pmd.h>
>  #include <rte_bus_ifpga.h>
>  #include <ifpga_logs.h>
> +#include <rte_pmd_i40e.h>
>  
>  #include "ipn3ke_rawdev_api.h"
>  #include "ipn3ke_flow.h"
> @@ -2918,8 +2919,11 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
>  	rpst->switch_domain_id = representor_param->switch_domain_id;
>  	rpst->port_id = representor_param->port_id;
>  	rpst->hw = representor_param->hw;
> -	rpst->i40e_pf_eth = NULL;
> -	rpst->i40e_pf_eth_port_id = 0xFFFF;
> +	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
> +	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
> +	if (rpst->i40e_pf_eth)
> +		rte_pmd_i40e_set_switch_dev(rpst->i40e_pf_eth_port_id,
> +					    rpst->ethdev);
>  
>  	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
>  	if (!ethdev->data->mac_addrs) {

Re: [dpdk-dev] [PATCH v9 12/18] net/ipn3ke: remove configuration for i40e port bonding

[Pei, Andy]

Hi Conole,

Thanks.
It will be included in next version.

-----Original Message-----
From: Aaron Conole [mailto:aconole@redhat.com] 
Sent: Monday, October 14, 2019 10:40 PM
To: Pei, Andy <andy.pei@intel.com>
Cc: dev@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>; Ye, Xiaolong <xiaolong.ye@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>
Subject: Re: [dpdk-dev] [PATCH v9 12/18] net/ipn3ke: remove configuration for i40e port bonding

Andy Pei <andy.pei@intel.com> writes:

> From: Rosen Xu <rosen.xu@intel.com>
>
> The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev, so 
> it doesn't need to provide configuration for i40e port bonding.
>
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/net/ipn3ke/Makefile             |   2 +

You'll also need to include a change to the meson file to ensure the include path will work for meson based builds.

>  drivers/net/ipn3ke/ipn3ke_ethdev.c      | 289 ++++----------------------------
>  drivers/net/ipn3ke/ipn3ke_representor.c |   8 +-
>  3 files changed, 44 insertions(+), 255 deletions(-)
>
> diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile 
> index 8c3ae37..2c65e49 100644
> --- a/drivers/net/ipn3ke/Makefile
> +++ b/drivers/net/ipn3ke/Makefile
> @@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API  CFLAGS += -O3  
> CFLAGS += $(WERROR_FLAGS)  CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
> +CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga CFLAGS += 
> +-I$(RTE_SDK)/drivers/net/i40e
>  LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring  LDLIBS += 
> -lrte_ethdev -lrte_net -lrte_kvargs  LDLIBS += -lrte_bus_ifpga diff 
> --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c 
> b/drivers/net/ipn3ke/ipn3ke_ethdev.c
> index 28d8aaf..3051cdf 100644
> --- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
> +++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
> @@ -19,6 +19,7 @@
>  #include <rte_bus_ifpga.h>
>  #include <ifpga_common.h>
>  #include <ifpga_logs.h>
> +#include <ifpga_rawdev.h>
>  
>  #include "ipn3ke_rawdev_api.h"
>  #include "ipn3ke_flow.h"
> @@ -324,7 +325,8 @@
>  				"LineSideMACType", &mac_type);
>  	hw->retimer.mac_type = (int)mac_type;
>  
> -	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
> +	hw->acc_tm = 0;
> +	hw->acc_flow = 0;
>  
>  	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
>  		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) { @@ -342,6 
> +344,12 @@
>  		/* After reset, wait until init done */
>  		if (ipn3ke_vbng_init_done(hw))
>  			return -1;
> +
> +		hw->acc_tm = 1;
> +		hw->acc_flow = 1;
> +
> +		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
> +			IPN3KE_READ_REG(hw, 0));
>  	}
>  
>  	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) 
> { @@ -409,9 +417,6 @@
>  		hw->flow_hw_enable = 1;
>  	}
>  
> -	hw->acc_tm = 0;
> -	hw->acc_flow = 0;
> -
>  	return 0;
>  }
>  
> @@ -462,7 +467,11 @@ static int ipn3ke_vswitch_probe(struct 
> rte_afu_device *afu_dev)  {
>  	char name[RTE_ETH_NAME_MAX_LEN];
>  	struct ipn3ke_hw *hw;
> -	int i, retval;
> +	struct rte_eth_dev *i40e_eth;
> +	struct ifpga_rawdev *ifpga_dev;
> +	uint16_t port_id;
> +	int i, j, retval;
> +	char *fvl_bdf;
>  
>  	/* check if the AFU device has been probed already */
>  	/* allocate shared mcp_vswitch structure */ @@ -489,7 +498,12 @@ 
> static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
>  	if (retval)
>  		return retval;
>  
> +	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
> +		if (!ifpga_dev)
> +			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
> +
>  	/* probe representor ports */
> +	j = 0;
>  	for (i = 0; i < hw->port_num; i++) {
>  		struct ipn3ke_rpst rpst = {
>  			.port_id = i,
> @@ -501,6 +515,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
>  		snprintf(name, sizeof(name), "net_%s_representor_%d",
>  			afu_dev->device.name, i);
>  
> +		for (; j < 8; j++) {
> +			fvl_bdf = ifpga_dev->fvl_bdf[j];
> +			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
> +				&port_id);
> +			if (retval) {
> +				continue;
> +			} else {
> +				i40e_eth = &rte_eth_devices[port_id];
> +				rpst.i40e_pf_eth = i40e_eth;
> +				rpst.i40e_pf_eth_port_id = port_id;
> +
> +				j++;
> +				break;
> +			}
> +		}
> +
>  		retval = rte_eth_dev_create(&afu_dev->device, name,
>  			sizeof(struct ipn3ke_rpst), NULL, NULL,
>  			ipn3ke_rpst_init, &rpst);
> @@ -508,6 +538,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
>  		if (retval)
>  			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
>  								name);
> +
>  	}
>  
>  	return 0;
> @@ -553,254 +584,6 @@ static int ipn3ke_vswitch_remove(struct 
> rte_afu_device *afu_dev)
>  
>  RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
>  
> -static const char * const valid_args[] = {
> -#define IPN3KE_AFU_NAME         "afu"
> -		IPN3KE_AFU_NAME,
> -#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
> -		IPN3KE_FPGA_ACCELERATION_LIST,
> -#define IPN3KE_I40E_PF_LIST     "i40e_pf"
> -		IPN3KE_I40E_PF_LIST,
> -		NULL
> -};
> -
> -static int
> -ipn3ke_cfg_parse_acc_list(const char *afu_name,
> -	const char *acc_list_name)
> -{
> -	struct rte_afu_device *afu_dev;
> -	struct ipn3ke_hw *hw;
> -	const char *p_source;
> -	char *p_start;
> -	char name[RTE_ETH_NAME_MAX_LEN];
> -
> -	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
> -	if (!afu_dev)
> -		return -1;
> -	hw = afu_dev->shared.data;
> -	if (!hw)
> -		return -1;
> -
> -	p_source = acc_list_name;
> -	while (*p_source) {
> -		while ((*p_source == '{') || (*p_source == '|'))
> -			p_source++;
> -		p_start = name;
> -		while ((*p_source != '|') && (*p_source != '}'))
> -			*p_start++ = *p_source++;
> -		*p_start = 0;
> -		if (!strcmp(name, "tm") && hw->tm_hw_enable)
> -			hw->acc_tm = 1;
> -
> -		if (!strcmp(name, "flow") && hw->flow_hw_enable)
> -			hw->acc_flow = 1;
> -
> -		if (*p_source == '}')
> -			return 0;
> -	}
> -
> -	return 0;
> -}
> -
> -static int
> -ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
> -	const char *pf_name)
> -{
> -	struct rte_eth_dev *i40e_eth, *rpst_eth;
> -	struct rte_afu_device *afu_dev;
> -	struct ipn3ke_rpst *rpst;
> -	struct ipn3ke_hw *hw;
> -	const char *p_source;
> -	char *p_start;
> -	char name[RTE_ETH_NAME_MAX_LEN];
> -	uint16_t port_id;
> -	int i;
> -	int ret = -1;
> -
> -	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
> -	if (!afu_dev)
> -		return -1;
> -	hw = afu_dev->shared.data;
> -	if (!hw)
> -		return -1;
> -
> -	p_source = pf_name;
> -	for (i = 0; i < hw->port_num; i++) {
> -		snprintf(name, sizeof(name), "net_%s_representor_%d",
> -			afu_name, i);
> -		ret = rte_eth_dev_get_port_by_name(name, &port_id);
> -		if (ret)
> -			return -1;
> -		rpst_eth = &rte_eth_devices[port_id];
> -		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
> -
> -		while ((*p_source == '{') || (*p_source == '|'))
> -			p_source++;
> -		p_start = name;
> -		while ((*p_source != '|') && (*p_source != '}'))
> -			*p_start++ = *p_source++;
> -		*p_start = 0;
> -
> -		ret = rte_eth_dev_get_port_by_name(name, &port_id);
> -		if (ret)
> -			return -1;
> -		i40e_eth = &rte_eth_devices[port_id];
> -
> -		rpst->i40e_pf_eth = i40e_eth;
> -		rpst->i40e_pf_eth_port_id = port_id;
> -
> -		if ((*p_source == '}') || !(*p_source))
> -			break;
> -	}
> -
> -	return 0;
> -}
> -
> -static int
> -ipn3ke_cfg_probe(struct rte_vdev_device *dev) -{
> -	struct rte_devargs *devargs;
> -	struct rte_kvargs *kvlist = NULL;
> -	char *afu_name = NULL;
> -	char *acc_name = NULL;
> -	char *pf_name = NULL;
> -	int afu_name_en = 0;
> -	int acc_list_en = 0;
> -	int pf_list_en = 0;
> -	int ret = -1;
> -
> -	devargs = dev->device.devargs;
> -
> -	kvlist = rte_kvargs_parse(devargs->args, valid_args);
> -	if (!kvlist) {
> -		IPN3KE_AFU_PMD_ERR("error when parsing param");
> -		goto end;
> -	}
> -
> -	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
> -		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
> -				       &rte_ifpga_get_string_arg,
> -				       &afu_name) < 0) {
> -			IPN3KE_AFU_PMD_ERR("error to parse %s",
> -				     IPN3KE_AFU_NAME);
> -			goto end;
> -		} else {
> -			afu_name_en = 1;
> -		}
> -	}
> -
> -	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
> -		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
> -				       &rte_ifpga_get_string_arg,
> -				       &acc_name) < 0) {
> -			IPN3KE_AFU_PMD_ERR("error to parse %s",
> -				     IPN3KE_FPGA_ACCELERATION_LIST);
> -			goto end;
> -		} else {
> -			acc_list_en = 1;
> -		}
> -	}
> -
> -	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
> -		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
> -				       &rte_ifpga_get_string_arg,
> -				       &pf_name) < 0) {
> -			IPN3KE_AFU_PMD_ERR("error to parse %s",
> -				     IPN3KE_I40E_PF_LIST);
> -			goto end;
> -		} else {
> -			pf_list_en = 1;
> -		}
> -	}
> -
> -	if (!afu_name_en) {
> -		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
> -			  IPN3KE_AFU_NAME);
> -		goto end;
> -	}
> -
> -	if (!pf_list_en) {
> -		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
> -			  IPN3KE_I40E_PF_LIST);
> -		goto end;
> -	}
> -
> -	if (acc_list_en) {
> -		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
> -		if (ret) {
> -			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
> -			  IPN3KE_FPGA_ACCELERATION_LIST);
> -			goto end;
> -		}
> -	} else {
> -		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
> -			  IPN3KE_FPGA_ACCELERATION_LIST);
> -	}
> -
> -	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
> -	if (ret)
> -		goto end;
> -end:
> -	if (kvlist)
> -		rte_kvargs_free(kvlist);
> -	if (afu_name)
> -		free(afu_name);
> -	if (acc_name)
> -		free(acc_name);
> -
> -	return ret;
> -}
> -
> -static int
> -ipn3ke_cfg_remove(struct rte_vdev_device *dev) -{
> -	struct rte_devargs *devargs;
> -	struct rte_kvargs *kvlist = NULL;
> -	char *afu_name = NULL;
> -	struct rte_afu_device *afu_dev;
> -	int ret = -1;
> -
> -	devargs = dev->device.devargs;
> -
> -	kvlist = rte_kvargs_parse(devargs->args, valid_args);
> -	if (!kvlist) {
> -		IPN3KE_AFU_PMD_ERR("error when parsing param");
> -		goto end;
> -	}
> -
> -	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
> -		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
> -				       &rte_ifpga_get_string_arg,
> -				       &afu_name) < 0) {
> -			IPN3KE_AFU_PMD_ERR("error to parse %s",
> -				     IPN3KE_AFU_NAME);
> -		} else {
> -			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
> -			if (!afu_dev)
> -				goto end;
> -			ret = ipn3ke_vswitch_remove(afu_dev);
> -		}
> -	} else {
> -		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
> -	}
> -
> -end:
> -	if (kvlist)
> -		rte_kvargs_free(kvlist);
> -
> -	return ret;
> -}
> -
> -static struct rte_vdev_driver ipn3ke_cfg_driver = {
> -	.probe = ipn3ke_cfg_probe,
> -	.remove = ipn3ke_cfg_remove,
> -};
> -
> -RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver); 
> -RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
> -	"afu=<string> "
> -	"fpga_acc=<string>"
> -	"i40e_pf=<string>");
> -
>  RTE_INIT(ipn3ke_afu_init_log)
>  {
>  	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
> diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c 
> b/drivers/net/ipn3ke/ipn3ke_representor.c
> index d37f5e2..7e5d29d 100644
> --- a/drivers/net/ipn3ke/ipn3ke_representor.c
> +++ b/drivers/net/ipn3ke/ipn3ke_representor.c
> @@ -20,6 +20,7 @@
>  #include <rte_rawdev_pmd.h>
>  #include <rte_bus_ifpga.h>
>  #include <ifpga_logs.h>
> +#include <rte_pmd_i40e.h>
>  
>  #include "ipn3ke_rawdev_api.h"
>  #include "ipn3ke_flow.h"
> @@ -2918,8 +2919,11 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
>  	rpst->switch_domain_id = representor_param->switch_domain_id;
>  	rpst->port_id = representor_param->port_id;
>  	rpst->hw = representor_param->hw;
> -	rpst->i40e_pf_eth = NULL;
> -	rpst->i40e_pf_eth_port_id = 0xFFFF;
> +	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
> +	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
> +	if (rpst->i40e_pf_eth)
> +		rte_pmd_i40e_set_switch_dev(rpst->i40e_pf_eth_port_id,
> +					    rpst->ethdev);
>  
>  	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
>  	if (!ethdev->data->mac_addrs) {

[dpdk-dev] [PATCH v9 13/18] raw/ifpga/base: add secure support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 136 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 +++++++++++++-----
 4 files changed, 197 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..e597e47 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,62 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager"))
+			sysmgr_offset = offset;
+			break;
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +575,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v9 13/18] raw/ifpga/base: add secure support

[Ye Xiaolong]

On 10/14, Andy Pei wrote:
>From: Tianfei zhang <tianfei.zhang@intel.com>
>
>Add secure max10 device support.
>
>Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
>Signed-off-by: Andy Pei <andy.pei@intel.com>
>---
> drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
> drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
> drivers/raw/ifpga/base/opae_intel_max10.c | 136 +++++++++++++++++++++++++-----
> drivers/raw/ifpga/base/opae_intel_max10.h |  80 +++++++++++++-----
> 4 files changed, 197 insertions(+), 47 deletions(-)
>
>+static int
>+max10_secure_hw_init(struct intel_max10_device *dev)
>+{
>+	int offset, sysmgr_offset = 0;
>+	char *fdt_root;
>+
>+	fdt_root = dev->fdt_root;
>+	if (!fdt_root) {
>+		dev_debug(dev, "skip init as not find Device Tree\n");
>+		return 0;
>+	}
>+
>+	fdt_for_each_subnode(offset, fdt_root, 0) {
>+		if (!fdt_node_check_compatible(fdt_root, offset,
>+					"intel-max10,system-manager"))
>+			sysmgr_offset = offset;
>+			break;
>+	}


Do you miss a brace for the if block? See compilation error below: 

dpdk/drivers/raw/ifpga/base/opae_intel_max10.c:541:3: error: this ‘if’ clause does not guard... [-Werror=misleading-indentation]
   if (!fdt_node_check_compatible(fdt_root, offset,
      ^~
      /home/yexl/workspace/dpdk/drivers/raw/ifpga/base/opae_intel_max10.c:544:4: note: ...this statement, but the latter is misleadingly indented as if it were guarded by the ‘if’
          break;
	      ^~~~~

Re: [dpdk-dev] [PATCH v9 13/18] raw/ifpga/base: add secure support

[Pei, Andy]

Hi, Xiaolong

Thanks.
This will be fixed in v10.

-----Original Message-----
From: Ye, Xiaolong 
Sent: Saturday, October 19, 2019 8:57 AM
To: Pei, Andy <andy.pei@intel.com>
Cc: dev@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>
Subject: Re: [PATCH v9 13/18] raw/ifpga/base: add secure support

On 10/14, Andy Pei wrote:
>From: Tianfei zhang <tianfei.zhang@intel.com>
>
>Add secure max10 device support.
>
>Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
>Signed-off-by: Andy Pei <andy.pei@intel.com>
>---
> drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
> drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
> drivers/raw/ifpga/base/opae_intel_max10.c | 136 
>+++++++++++++++++++++++++-----  
>drivers/raw/ifpga/base/opae_intel_max10.h |  80 +++++++++++++-----
> 4 files changed, 197 insertions(+), 47 deletions(-)
>
>+static int
>+max10_secure_hw_init(struct intel_max10_device *dev) {
>+	int offset, sysmgr_offset = 0;
>+	char *fdt_root;
>+
>+	fdt_root = dev->fdt_root;
>+	if (!fdt_root) {
>+		dev_debug(dev, "skip init as not find Device Tree\n");
>+		return 0;
>+	}
>+
>+	fdt_for_each_subnode(offset, fdt_root, 0) {
>+		if (!fdt_node_check_compatible(fdt_root, offset,
>+					"intel-max10,system-manager"))
>+			sysmgr_offset = offset;
>+			break;
>+	}


Do you miss a brace for the if block? See compilation error below: 

dpdk/drivers/raw/ifpga/base/opae_intel_max10.c:541:3: error: this ‘if’ clause does not guard... [-Werror=misleading-indentation]
   if (!fdt_node_check_compatible(fdt_root, offset,
      ^~
      /home/yexl/workspace/dpdk/drivers/raw/ifpga/base/opae_intel_max10.c:544:4: note: ...this statement, but the latter is misleadingly indented as if it were guarded by the ‘if’
          break;
	      ^~~~~

[dpdk-dev] [PATCH v9 14/18] raw/ifpga/base: configure FEC mode

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 15/18] raw/ifpga/base: clean fme errors

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occurred.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 5d6d630..5905eac 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 01ff76a..95f079a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1174,6 +1174,25 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1183,6 +1202,9 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 16/18] raw/ifpga/base: add new API get board info

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..9f0147d 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disaggregate;
+	u8 lightweight;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..799d67d 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweight = id.lightweiht;
+	fme->board_info.disaggregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweight:%s\n"
+			"disaggregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweight),
+			check_support(fme->board_info.disaggregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 17/18] raw/ifpga: add lightweight fpga image support

[Andy Pei]

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 95f079a..e87af66 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -835,6 +835,8 @@ static int set_surprise_link_check_aer(
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -861,22 +863,40 @@ static int set_surprise_link_check_aer(
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweight) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
-		uuid.b + 8, sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v9 18/18] raw/ifpga/base: add multiple cards support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, there is one MAX10 chip in each card, and
all of the sensors are connected to MAX10 chip. To support multiple
cards in one server, we introducing a sensor device list under
intel_max10_device instead of a global list. On the other hand, we
using separate intel_max10_device instance for each opae_adatper.

Add mutex lock on do_transaction() function for SPI driver to avoid
race condition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c            |  40 +++++++---
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          |  14 ++--
 drivers/raw/ifpga/base/opae_hw_api.h          |  15 ++--
 drivers/raw/ifpga/base/opae_i2c.c             |  44 ++++++++---
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_intel_max10.c     | 110 ++++++++++++++------------
 drivers/raw/ifpga/base/opae_intel_max10.h     |  19 +++--
 drivers/raw/ifpga/base/opae_spi.c             |   5 --
 drivers/raw/ifpga/base/opae_spi.h             |   3 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  44 +++++++++--
 drivers/raw/ifpga/ifpga_rawdev.c              |  14 ++--
 12 files changed, 205 insertions(+), 109 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 799d67d..c31a94c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -839,8 +839,13 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	struct ifpga_hw *hw;
 	u32 val;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
@@ -854,7 +859,10 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	fme->board_info.seu = id.seu;
 	fme->board_info.ptp = id.ptp;
 
-	dev_info(fme, "found: board: %s type: %s\n",
+	dev_info(fme, "found: PCI dev: %02x:%02x:%x board: %s type: %s\n",
+			hw->pci_data->bus,
+			hw->pci_data->devid,
+			hw->pci_data->function,
 			board_major_to_string(fme->board_info.major),
 			board_type_to_string(fme->board_info.type));
 
@@ -882,11 +890,11 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
-	if (max10_sys_read(MAX10_BUILD_VER, &val))
+	if (max10_sys_read(fme->max10_dev, MAX10_BUILD_VER, &val))
 		return -EINVAL;
 	fme->board_info.max10_version = val & 0xffffff;
 
-	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+	if (max10_sys_read(fme->max10_dev, NIOS2_FW_VERSION, &val))
 		return -EINVAL;
 	fme->board_info.nios_fw_version = val & 0xffffff;
 
@@ -897,12 +905,12 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	return 0;
 }
 
-static int spi_self_checking(void)
+static int spi_self_checking(struct intel_max10_device *dev)
 {
 	u32 val;
 	int ret;
 
-	ret = max10_sys_read(MAX10_TEST_REG, &val);
+	ret = max10_sys_read(dev, MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
@@ -937,10 +945,11 @@ static int fme_spi_init(struct ifpga_feature *feature)
 		goto spi_fail;
 	}
 
+
 	fme->max10_dev = max10;
 
 	/* SPI self test */
-	if (spi_self_checking()) {
+	if (spi_self_checking(max10)) {
 		ret = -EIO;
 		goto max10_fail;
 	}
@@ -1041,8 +1050,18 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 	struct ifpga_fme_hw *fme = (struct ifpga_fme_hw *)feature->parent;
 	struct altera_spi_device *spi_master;
 	struct intel_max10_device *max10;
+	struct ifpga_hw *hw;
+	struct opae_manager *mgr;
 	int ret = 0;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
+	mgr = hw->adapter->mgr;
+	if (!mgr)
+		return -ENODEV;
+
 	dev_info(fme, "FME SPI Master (NIOS) Init.\n");
 	dev_debug(fme, "FME SPI base addr %p.\n",
 			feature->addr);
@@ -1080,12 +1099,15 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 		goto release_dev;
 	}
 
+	max10->bus = hw->pci_data->bus;
+
 	fme_get_board_interface(fme);
 
 	fme->max10_dev = max10;
+	mgr->sensor_list = &max10->opae_sensor_list;
 
 	/* SPI self test */
-	if (spi_self_checking())
+	if (spi_self_checking(max10))
 		goto spi_fail;
 
 	return ret;
@@ -1344,7 +1366,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(dev, PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1372,7 +1394,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(sensor->value_reg, value)) {
+	if (max10_sys_read(dev, sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_debug.c b/drivers/raw/ifpga/base/opae_debug.c
index 88f2d5c..dad3ea3 100644
--- a/drivers/raw/ifpga/base/opae_debug.c
+++ b/drivers/raw/ifpga/base/opae_debug.c
@@ -59,6 +59,9 @@ static void opae_adapter_data_dump(void *data)
 		opae_log("OPAE Adapter Type = PCI\n");
 		opae_log("PCI Device ID: 0x%04x\n", d_pci->device_id);
 		opae_log("PCI Vendor ID: 0x%04x\n", d_pci->vendor_id);
+		opae_log("PCI bus: 0x%04x\n", d_pci->bus);
+		opae_log("PCI devid: 0x%04x\n", d_pci->devid);
+		opae_log("PCI function: 0x%04x\n", d_pci->function);
 
 		for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 			r = &d_pci->region[i];
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 1ccc967..c969dfe 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -583,11 +583,12 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_id(unsigned int id)
+opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (sensor->id == id)
 			return sensor;
 
@@ -601,11 +602,12 @@ struct opae_sensor_info *
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_name(const char *name)
+opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (!strcmp(sensor->name, name))
 			return sensor;
 
@@ -630,7 +632,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_name(name);
+	sensor = opae_mgr_get_sensor_by_name(mgr, name);
 	if (!sensor)
 		return -ENODEV;
 
@@ -658,7 +660,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_id(id);
+	sensor = opae_mgr_get_sensor_by_id(mgr, id);
 	if (!sensor)
 		return -ENODEV;
 
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index b78fbd5..cdf369f 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -40,6 +40,7 @@ struct opae_manager {
 	struct opae_adapter *adapter;
 	struct opae_manager_ops *ops;
 	struct opae_manager_networking_ops *network_ops;
+	struct opae_sensor_list *sensor_list;
 	void *data;
 };
 
@@ -75,9 +76,8 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
-extern struct opae_sensor_list opae_sensor_list;
-#define opae_mgr_for_each_sensor(sensor) \
-	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+#define opae_mgr_for_each_sensor(mgr, sensor) \
+	TAILQ_FOREACH(sensor, mgr->sensor_list, node)
 
 /* OPAE Manager APIs */
 struct opae_manager *
@@ -88,8 +88,10 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
-struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
-struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id);
 int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
 		const char *name, unsigned int *value);
 int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
@@ -241,6 +243,9 @@ struct opae_adapter_data_pci {
 	enum opae_adapter_type type;
 	u16 device_id;
 	u16 vendor_id;
+	u16 bus; /*Device bus for PCI */
+	u16 devid; /* Device ID */
+	u16 function; /* Device function */
 	struct opae_reg_region region[PCI_MAX_RESOURCE];
 	int vfio_dev_fd;  /* VFIO device file descriptor */
 };
diff --git a/drivers/raw/ifpga/base/opae_i2c.c b/drivers/raw/ifpga/base/opae_i2c.c
index f8bc247..846d751 100644
--- a/drivers/raw/ifpga/base/opae_i2c.c
+++ b/drivers/raw/ifpga/base/opae_i2c.c
@@ -28,6 +28,9 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 {
 	u8 msgbuf[2];
 	int i = 0;
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
 
 	if (flags & I2C_FLAG_ADDR16)
 		msgbuf[i++] = offset >> 8;
@@ -49,10 +52,16 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		},
 	};
 
-	if (!dev->xfer)
-		return -ENODEV;
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	ret = i2c_transfer(dev, msg, 2);
 
-	return i2c_transfer(dev, msg, 2);
+exit:
+	pthread_mutex_unlock(&dev->lock);
+	return ret;
 }
 
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
@@ -63,12 +72,18 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	int ret;
 	int i = 0;
 
-	if (!dev->xfer)
-		return -ENODEV;
+	pthread_mutex_lock(&dev->lock);
+
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
 
 	buf = opae_malloc(I2C_MAX_OFFSET_LEN + len);
-	if (!buf)
-		return -ENOMEM;
+	if (!buf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
 
 	msg.addr = slave_addr;
 	msg.flags = 0;
@@ -84,6 +99,8 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	ret = i2c_transfer(dev, &msg, 1);
 
 	opae_free(buf);
+exit:
+	pthread_mutex_unlock(&dev->lock);
 	return ret;
 }
 
@@ -477,14 +494,19 @@ struct altera_i2c_dev *altera_i2c_probe(void *base)
 	dev->i2c_clk = dev->i2c_param.ref_clk * 1000000;
 	dev->xfer = altera_i2c_xfer;
 
+	if (pthread_mutex_init(&dev->lock, NULL))
+		return NULL;
+
 	altera_i2c_hardware_init(dev);
 
 	return dev;
 }
 
-int altera_i2c_remove(struct altera_i2c_dev *dev)
+void altera_i2c_remove(struct altera_i2c_dev *dev)
 {
-	altera_i2c_disable(dev);
-
-	return 0;
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
+		altera_i2c_disable(dev);
+		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/base/opae_i2c.h b/drivers/raw/ifpga/base/opae_i2c.h
index 8890c8f..266e127 100644
--- a/drivers/raw/ifpga/base/opae_i2c.h
+++ b/drivers/raw/ifpga/base/opae_i2c.h
@@ -93,6 +93,7 @@ struct altera_i2c_dev {
 	u32 isr_mask;
 	u8 *buf;
 	int (*xfer)(struct altera_i2c_dev *dev, struct i2c_msg *msg, int num);
+	pthread_mutex_t lock;
 };
 
 /**
@@ -114,7 +115,7 @@ enum i2c_msg_flags {
 };
 
 struct altera_i2c_dev *altera_i2c_probe(void *base);
-int altera_i2c_remove(struct altera_i2c_dev *dev);
+void altera_i2c_remove(struct altera_i2c_dev *dev);
 int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		u32 offset, u8 *buf, u32 count);
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index e597e47..680a580 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -5,45 +5,50 @@
 #include "opae_intel_max10.h"
 #include <libfdt.h>
 
-static struct intel_max10_device *g_max10;
-
-struct opae_sensor_list opae_sensor_list =
-	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
-
-int max10_reg_read(unsigned int reg, unsigned int *val)
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_read(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x\n", __func__, dev->bus, reg);
+
+	return spi_transaction_read(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)val);
 }
 
-int max10_reg_write(unsigned int reg, unsigned int val)
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val)
 {
 	unsigned int tmp = val;
 
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_write(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x, val:0x%x\n", __func__,
+			dev->bus, reg, val);
+
+	return spi_transaction_write(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)&tmp);
 }
 
-int max10_sys_read(unsigned int offset, unsigned int *val)
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_read(g_max10->base + offset, val);
+
+	return max10_reg_read(dev, dev->base + offset, val);
 }
 
-int max10_sys_write(unsigned int offset, unsigned int val)
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_write(g_max10->base + offset, val);
+	return max10_reg_write(dev, dev->base + offset, val);
 }
 
 static struct max10_compatible_id max10_id_table[] = {
@@ -86,8 +91,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
-static int altera_nor_flash_read(u32 offset,
-		void *buffer, u32 len)
+static int altera_nor_flash_read(struct intel_max10_device *dev,
+	u32 offset, void *buffer, u32 len)
 {
 	int word_len;
 	int i;
@@ -95,13 +100,13 @@ static int altera_nor_flash_read(u32 offset,
 	unsigned int value;
 	int ret;
 
-	if (!buffer || len <= 0)
+	if (!dev || !buffer || len <= 0)
 		return -ENODEV;
 
 	word_len = len/4;
 
 	for (i = 0; i < word_len; i++) {
-		ret = max10_reg_read(offset + i*4,
+		ret = max10_reg_read(dev, offset + i*4,
 				&value);
 		if (ret)
 			return -EBUSY;
@@ -112,12 +117,12 @@ static int altera_nor_flash_read(u32 offset,
 	return 0;
 }
 
-static int enable_nor_flash(bool on)
+static int enable_nor_flash(struct intel_max10_device *dev, bool on)
 {
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_sys_read(RSU_REG, &val);
+	ret = max10_sys_read(dev, RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -128,7 +133,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_sys_write(RSU_REG, val);
+	return max10_sys_write(dev, RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -140,7 +145,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_sys_read(DT_AVAIL_REG, &val);
+	ret = max10_sys_read(max10, DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -151,19 +156,19 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
+	ret = max10_sys_read(max10, DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
 	}
 
-	ret = enable_nor_flash(true);
+	ret = enable_nor_flash(max10, true);
 	if (ret) {
 		dev_err(max10 "fail to enable flash\n");
 		return ret;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	ret = altera_nor_flash_read(max10, dt_addr, &hdr, sizeof(hdr));
 	if (ret) {
 		dev_err(max10 "read fdt header fail\n");
 		goto done;
@@ -188,7 +193,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		goto done;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	ret = altera_nor_flash_read(max10, dt_addr, fdt_root, dt_size);
 	if (ret) {
 		dev_err(max10 "cannot read device table\n");
 		goto done;
@@ -207,7 +212,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	max10->fdt_root = fdt_root;
 
 done:
-	ret = enable_nor_flash(false);
+	ret = enable_nor_flash(max10, false);
 
 	if (ret && fdt_root)
 		opae_free(fdt_root);
@@ -298,12 +303,12 @@ static int fdt_get_named_reg(const void *fdt, int node, const char *name,
 	return fdt_get_reg(fdt, node, idx, start, size);
 }
 
-static void max10_sensor_uinit(void)
+static void max10_sensor_uinit(struct intel_max10_device *dev)
 {
 	struct opae_sensor_info *info;
 
-	TAILQ_FOREACH(info, &opae_sensor_list, node) {
-		TAILQ_REMOVE(&opae_sensor_list, info, node);
+	TAILQ_FOREACH(info, &dev->opae_sensor_list, node) {
+		TAILQ_REMOVE(&dev->opae_sensor_list, info, node);
 		opae_free(info);
 	}
 }
@@ -313,8 +318,8 @@ static bool sensor_reg_valid(struct sensor_reg *reg)
 	return !!reg->size;
 }
 
-static int max10_add_sensor(struct raw_sensor_info *info,
-		struct opae_sensor_info *sensor)
+static int max10_add_sensor(struct intel_max10_device *dev,
+	struct raw_sensor_info *info, struct opae_sensor_info *sensor)
 {
 	int i;
 	int ret = 0;
@@ -332,12 +337,15 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_sys_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(dev, info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
-		if (val == 0xdeadbeef)
+		if (val == 0xdeadbeef) {
+			dev_debug(dev, "%s: sensor:%s invalid 0x%x at:%d\n",
+				__func__, sensor->name, val, i);
 			continue;
+		}
 
 		val *= info->multiplier;
 
@@ -458,7 +466,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
 		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
 
-		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+		dev_debug(dev, "found sensor from DTB: %s: %s: %u: %u\n",
 				raw->name, raw->type,
 				raw->id, raw->multiplier);
 
@@ -473,15 +481,16 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 			goto free_sensor;
 		}
 
-		if (max10_add_sensor(raw, sensor)) {
+		if (max10_add_sensor(dev, raw, sensor)) {
 			ret = -EINVAL;
 			opae_free(sensor);
 			goto free_sensor;
 		}
 
-		if (sensor->flags & OPAE_SENSOR_VALID)
-			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
-		else
+		if (sensor->flags & OPAE_SENSOR_VALID) {
+			TAILQ_INSERT_TAIL(&dev->opae_sensor_list, sensor, node);
+			dev_info(dev, "found valid sensor: %s\n", sensor->name);
+		} else
 			opae_free(sensor);
 
 		opae_free(raw);
@@ -492,7 +501,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 free_sensor:
 	if (raw)
 		opae_free(raw);
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 	return ret;
 }
 
@@ -500,7 +509,7 @@ static int check_max10_version(struct intel_max10_device *dev)
 {
 	unsigned int v;
 
-	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+	if (!max10_reg_read(dev, MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
 				&v)) {
 		if (v != 0xffffffff) {
 			dev_info(dev, "secure MAX10 detected\n");
@@ -564,6 +573,8 @@ struct intel_max10_device *
 	if (!dev)
 		return NULL;
 
+	TAILQ_INIT(&dev->opae_sensor_list);
+
 	dev->spi_master = spi;
 
 	dev->spi_tran_dev = spi_transaction_init(spi, chipselect);
@@ -572,9 +583,6 @@ struct intel_max10_device *
 		goto free_dev;
 	}
 
-	/* set the max10 device firstly */
-	g_max10 = dev;
-
 	/* check the max10 version */
 	ret = check_max10_version(dev);
 	if (ret) {
@@ -600,7 +608,7 @@ struct intel_max10_device *
 	}
 
 	/* read FPGA loading information */
-	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
+	ret = max10_sys_read(dev, FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
 		goto release_max10_hw;
@@ -610,14 +618,13 @@ struct intel_max10_device *
 	return dev;
 
 release_max10_hw:
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return NULL;
@@ -628,7 +635,7 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
@@ -636,7 +643,6 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index e632941..123cdc4 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -26,6 +26,9 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SECURE		BIT(6)
 #define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
@@ -33,6 +36,8 @@ struct intel_max10_device {
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
 	unsigned int base; /* max10 base address */
+	u16 bus;
+	struct opae_sensor_list opae_sensor_list;
 };
 
 /* retimer speed */
@@ -136,17 +141,19 @@ struct opae_retimer_status {
 
 #define DFT_MAX_SIZE		0x7e0000
 
-int max10_reg_read(unsigned int reg, unsigned int *val);
-int max10_reg_write(unsigned int reg, unsigned int val);
-int max10_sys_read(unsigned int offset, unsigned int *val);
-int max10_sys_write(unsigned int offset, unsigned int val);
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val);
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val);
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val);
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
-/** List of opae sensors */
-TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
 
 #define SENSOR_REG_VALUE 0x0
 #define SENSOR_REG_HIGH_WARN 0x1
diff --git a/drivers/raw/ifpga/base/opae_spi.c b/drivers/raw/ifpga/base/opae_spi.c
index cc52782..bfdc83e 100644
--- a/drivers/raw/ifpga/base/opae_spi.c
+++ b/drivers/raw/ifpga/base/opae_spi.c
@@ -181,7 +181,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 	u32 rxd;
 	unsigned int tx_data;
 	u32 status;
-	int retry = 0;
 	int ret;
 
 	while (count < dev->len) {
@@ -194,10 +193,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 				return -EIO;
 			if (status & ALTERA_SPI_STATUS_RRDY_MSK)
 				break;
-			if (retry++ > SPI_MAX_RETRY) {
-				dev_err(dev, "%s, read timeout\n", __func__);
-				return -EBUSY;
-			}
 		}
 
 		ret = spi_reg_read(dev, ALTERA_SPI_RXDATA, &rxd);
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index 6355deb..d20a4c3 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -38,7 +38,7 @@
 #define SPI_WRITE 0x20
 #define WRITE_DATA_MASK GENMASK_ULL(31, 0)
 
-#define SPI_MAX_RETRY 100000
+#define SPI_MAX_RETRY 1000000
 
 #define TYPE_SPI 0
 #define TYPE_NIOS_SPI 1
@@ -102,6 +102,7 @@ struct spi_transaction_dev {
 	struct altera_spi_device *dev;
 	int chipselect;
 	struct spi_tran_buffer *buffer;
+	pthread_mutex_t lock;
 };
 
 struct spi_tran_header {
diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 06ca625..013efee 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -154,6 +154,8 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	unsigned int rx_len = 0;
 	int retry = 0;
 	int spi_flags;
+	unsigned long timeout = msecs_to_timer_cycles(1000);
+	unsigned long ticks;
 	unsigned int resp_max_len = 2 * resp_len;
 
 	print_buffer("before bytes:", send_data, send_len);
@@ -207,8 +209,11 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	if (ret != SPI_FOUND_EOP) {
 		tx_buffer = NULL;
 		tx_len = 0;
-		if (retry++ > 10) {
-			dev_err(NULL, "cannot found valid data from SPI\n");
+		ticks = rte_get_timer_cycles();
+		if (time_after(ticks, timeout) &&
+				retry++ > SPI_MAX_RETRY) {
+			dev_err(NULL, "Have retry %d, found invalid packet data\n",
+				retry);
 			return -EBUSY;
 		}
 
@@ -427,23 +432,36 @@ static int do_transaction(struct spi_transaction_dev *dev, unsigned int addr,
 int spi_transaction_read(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_READ : SPI_TRAN_NON_SEQ_READ);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 int spi_transaction_write(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_WRITE : SPI_TRAN_NON_SEQ_WRITE);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 		int chipselect)
 {
 	struct spi_transaction_dev *spi_tran_dev;
+	int ret;
 
 	spi_tran_dev = opae_malloc(sizeof(struct spi_transaction_dev));
 	if (!spi_tran_dev)
@@ -453,18 +471,28 @@ struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 	spi_tran_dev->chipselect = chipselect;
 
 	spi_tran_dev->buffer = opae_malloc(sizeof(struct spi_tran_buffer));
-	if (!spi_tran_dev->buffer) {
-		opae_free(spi_tran_dev);
-		return NULL;
+	if (!spi_tran_dev->buffer)
+		goto err;
+
+	ret = pthread_mutex_init(&spi_tran_dev->lock, NULL);
+	if (ret) {
+		dev_err(spi_tran_dev, "fail to init mutex lock\n");
+		goto err;
 	}
 
 	return spi_tran_dev;
+
+err:
+	opae_free(spi_tran_dev);
+	return NULL;
 }
 
 void spi_transaction_remove(struct spi_transaction_dev *dev)
 {
 	if (dev && dev->buffer)
 		opae_free(dev->buffer);
-	if (dev)
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
 		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index e87af66..7253644 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -360,7 +360,7 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 	if (!mgr)
 		return -ENODEV;
 
-	opae_mgr_for_each_sensor(sensor) {
+	opae_mgr_for_each_sensor(mgr, sensor) {
 		if (!(sensor->flags & OPAE_SENSOR_VALID))
 			goto fail;
 
@@ -369,8 +369,8 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 			goto fail;
 
 		if (value == 0xdeadbeef) {
-			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
-					sensor->name, value);
+			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value %x\n",
+					raw_dev->dev_id, sensor->name, value);
 			continue;
 		}
 
@@ -393,8 +393,9 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 		/* monitor 12V AUX sensor */
 		if (!strcmp(sensor->name, "12V AUX Voltage")) {
 			if (value < AUX_VOLTAGE_WARN) {
-				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
-						sensor->name, value);
+				IFPGA_RAWDEV_PMD_INFO(
+					"%s reach theshold %d mV\n",
+					sensor->name, value);
 				*gsd_start = true;
 				break;
 			}
@@ -1433,6 +1434,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->bus = pci_dev->addr.bus;
+	data->devid = pci_dev->addr.devid;
+	data->function = pci_dev->addr.function;
 	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v9 00/18] add PCIe AER disable and IRQ support for ipn3ke

[Ye Xiaolong]

Hi, Andy

I saw there are several compilation issues reported in patchwork, could you
check it?

Thanks,
Xiaolong

On 10/14, Andy Pei wrote:
>This patch set adds PCIe AER disable and IRQ support for ipn3ke.
>Disable PCIe AER is very useful when FPGA reload. IRQ is used very
>widely in interrupt process.
>
>For ipn3ke is connect to CPU with PCIe switch, driver needs to scan
>all PCIe devices of ipn3ke, it also can get all i40e of card, so
>ipn3ke driver doesn't need to take some configuration of i40e.
>
>
>v9 updates:
>=========
>- Add mutex lock on do_transaction() function for SPI driver to avoid race 
>  condition.
>
>v8 updates:
>=========
>- add multiple cards support.
>
>v7 updates:
>==========
>- rename function i40e_set_switch_dev to rte_pmd_i40e_set_switch_dev
>  and move it to rte_pmd_i40e.c since it is declared at rte_pmd_i40e.h
>- function rte_pmd_i40e_set_switch_dev works as an external API,
>  use port_id but not rte_eth_dev as parameter.
>- add doxygen header here for the new API.
>- update the rte_pmd_i40e_version.map.
>- fix coding style issue.
>- enable CONFIG_RTE_EAL_VFIO in linux environment to build irq support.
>- for functions with a lot of similarity, extract out common function to reduce
>  duplication.
>
>v6 updates:
>=========
>- correct author information.
>- correct typo in commit message and remove Gerrit Change-Id's before
>  submitting upstream
>
>v5 updates:
>==========
>- add lightweight fpga image support. in lightweight fpga image mode,
>  ipn3ke representor will not be probed.
>
>v4 updates:
>===========
>- align with new naming standard.
>
>v3 updates:
>===========
>- Add FPGA network side port MTU configuration
>
>v2 updates:
>===========
>- Add AUX feature support
>
>Andy Pei (2):
>  net/i40e: i40e support ipn3ke FPGA port bonding
>  raw/ifpga: add lightweight fpga image support
>
>Rosen Xu (3):
>  raw/ifpga: add SEU error handler
>  raw/ifpga: add PCIe BDF devices tree scan
>  net/ipn3ke: remove configuration for i40e port bonding
>
>Tianfei zhang (13):
>  raw/ifpga/base: add irq support
>  raw/ifpga/base: clear pending bit
>  raw/ifpga/base: add SEU error support
>  raw/ifpga/base: add device tree support
>  raw/ifpga/base: align the send buffer for SPI
>  raw/ifpga/base: add sensor support
>  raw/ifpga/base: introducing sensor APIs
>  raw/ifpga/base: update SEU register definition
>  raw/ifpga/base: add secure support
>  raw/ifpga/base: configure FEC mode
>  raw/ifpga/base: clean fme errors
>  raw/ifpga/base: add new API get board info
>  raw/ifpga/base: add multiple cards support
>
> config/common_base                            |   4 +-
> config/common_linux                           |   6 +
> drivers/net/i40e/base/i40e_type.h             |   3 +
> drivers/net/i40e/i40e_ethdev.c                |   6 +
> drivers/net/i40e/rte_pmd_i40e.c               |  21 +
> drivers/net/i40e/rte_pmd_i40e.h               |  18 +
> drivers/net/i40e/rte_pmd_i40e_version.map     |   8 +-
> drivers/net/ipn3ke/Makefile                   |   2 +
> drivers/net/ipn3ke/ipn3ke_ethdev.c            | 289 ++-------
> drivers/net/ipn3ke/ipn3ke_representor.c       |   8 +-
> drivers/raw/ifpga/base/ifpga_api.c            |  21 +
> drivers/raw/ifpga/base/ifpga_defines.h        |  75 ++-
> drivers/raw/ifpga/base/ifpga_feature_dev.c    |  59 ++
> drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
> drivers/raw/ifpga/base/ifpga_fme.c            | 166 ++++-
> drivers/raw/ifpga/base/ifpga_fme_error.c      |  74 ++-
> drivers/raw/ifpga/base/ifpga_hw.h             |   2 +-
> drivers/raw/ifpga/base/ifpga_port.c           |  18 +
> drivers/raw/ifpga/base/ifpga_port_error.c     |  19 +
> drivers/raw/ifpga/base/opae_debug.c           |   3 +
> drivers/raw/ifpga/base/opae_hw_api.c          | 137 ++++
> drivers/raw/ifpga/base/opae_hw_api.h          |  26 +
> drivers/raw/ifpga/base/opae_i2c.c             |  44 +-
> drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
> drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
> drivers/raw/ifpga/base/opae_intel_max10.c     | 598 +++++++++++++++++-
> drivers/raw/ifpga/base/opae_intel_max10.h     | 157 ++++-
> drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
> drivers/raw/ifpga/base/opae_spi.c             |   5 -
> drivers/raw/ifpga/base/opae_spi.h             |  26 +-
> drivers/raw/ifpga/base/opae_spi_transaction.c |  84 ++-
> drivers/raw/ifpga/ifpga_rawdev.c              | 860 +++++++++++++++++++++++++-
> drivers/raw/ifpga/ifpga_rawdev.h              |  16 +
> mk/rte.app.mk                                 |   2 +-
> 34 files changed, 2348 insertions(+), 424 deletions(-)
>
>-- 
>1.8.3.1
>

[dpdk-dev] [PATCH v8 02/18] raw/ifpga/base: add irq support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME global error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

To build this feature, CONFIG_RTE_EAL_VFIO should be enabled.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                         |  2 +-
 config/common_linux                        |  6 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 59 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 19 ++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 18 +++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 19 ++++++++++
 6 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/config/common_base b/config/common_base
index e843a21..68a4f70 100644
--- a/config/common_base
+++ b/config/common_base
@@ -772,7 +772,7 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
 #
 # Compile PMD for Intel FPGA raw device
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
 
 #
 # Compile PMD for Intel IOAT raw device
diff --git a/config/common_linux b/config/common_linux
index 96e2e1f..a78b8c6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -68,3 +68,9 @@ CONFIG_RTE_LIBRTE_HINIC_PMD=y
 # Hisilicon HNS3 PMD driver
 #
 CONFIG_RTE_LIBRTE_HNS3_PMD=y
+
+#
+# Compile PMD for Intel FPGA raw device
+# To compile, CONFIG_RTE_EAL_VFIO should be enabled.
+#
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..0f852a7 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,61 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+#define FPGA_MAX_MSIX_VEC_COUNT	128
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MAX_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	/* 0 < irq_set->count < FPGA_MAX_MSIX_VEC_COUNT */
+	irq_set->count = count ?
+		(count > FPGA_MAX_MSIX_VEC_COUNT ?
+		 FPGA_MAX_MSIX_VEC_COUNT : count) : 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..2978c79 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,28 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&fme->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..c0aaf01 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,27 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..189f762 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,28 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 03/18] raw/ifpga/base: clear pending bit

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 2978c79..be041ec 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 04/18] raw/ifpga/base: add SEU error support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 31 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index be041ec..5d6d630 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,33 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr(struct ifpga_fme_hw *fme,
+		u64 *val, bool high)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+				FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	if (high)
+		*val = readq(&fme_err->seu_emr_h);
+	else
+		*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +297,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr(fme, &prop->data, 0);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr(fme, &prop->data, 1);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 05/18] raw/ifpga/base: add device tree support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

To eliminate build error, we currently do not compile raw/ifpga
and net/ipn3ke. User should install libfdt and libfdt-devel first,
modify config/common_linux, CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
to CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y, modify config/common_base,
CONFIG_RTE_LIBRTE_IPN3KE_PMD=y to CONFIG_RTE_LIBRTE_IPN3KE_PMD=n.
Then this function can work.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                        |   2 +-
 config/common_linux                       |   2 +-
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 mk/rte.app.mk                             |   2 +-
 5 files changed, 196 insertions(+), 3 deletions(-)

diff --git a/config/common_base b/config/common_base
index 68a4f70..c0c9d5e 100644
--- a/config/common_base
+++ b/config/common_base
@@ -336,7 +336,7 @@ CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
 #
 # Compile burst-oriented IPN3KE PMD driver
 #
-CONFIG_RTE_LIBRTE_IPN3KE_PMD=y
+CONFIG_RTE_LIBRTE_IPN3KE_PMD=n
 
 #
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
diff --git a/config/common_linux b/config/common_linux
index a78b8c6..c5cf3d6 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -73,4 +73,4 @@ CONFIG_RTE_LIBRTE_HNS3_PMD=y
 # Compile PMD for Intel FPGA raw device
 # To compile, CONFIG_RTE_EAL_VFIO should be enabled.
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index b91273f..28f3ef3 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -321,7 +321,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 06/18] raw/ifpga/base: align the send buffer for SPI

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 07/18] raw/ifpga/base: add sensor support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 08/18] raw/ifpga/base: introducing sensor APIs

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 09/18] raw/ifpga/base: update SEU register definition

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 10/18] raw/ifpga: add SEU error handler

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 245 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 1825143..f5e6119 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -27,6 +27,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -605,6 +607,236 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -652,6 +884,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -677,6 +910,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -696,6 +933,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -720,6 +958,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 11/18] raw/ifpga: add PCIe BDF devices tree scan

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 551 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 562 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index f5e6119..01ff76a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_malloc.h>
@@ -17,7 +19,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -25,6 +27,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -37,6 +40,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -64,6 +73,494 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(
+		struct ifpga_rawdev *ifpga_rdev, int force_disable);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+		int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
+					sensor->name, value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(
+	struct ifpga_rawdev *ifpga_rdev, int force_disable)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable || force_disable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev,
+					gsd_enable);
+				if (ret == 1 && !gsd_enable) {
+					gsd_enable = 1;
+					i = -1;
+				}
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -372,8 +869,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -842,6 +1340,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -855,7 +1354,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -869,6 +1368,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -987,6 +1494,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1018,13 +1526,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1038,7 +1565,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1065,6 +1593,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 12/18] net/ipn3ke: remove configuration for i40e port bonding

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/ipn3ke/Makefile             |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c      | 289 ++++----------------------------
 drivers/net/ipn3ke/ipn3ke_representor.c |   8 +-
 3 files changed, 44 insertions(+), 255 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..2c65e49 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index 28d8aaf..3051cdf 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -324,7 +325,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -342,6 +344,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -409,9 +417,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -462,7 +467,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -489,7 +498,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -501,6 +515,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -508,6 +538,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -553,254 +584,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index d37f5e2..7e5d29d 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2918,8 +2919,11 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		rte_pmd_i40e_set_switch_dev(rpst->i40e_pf_eth_port_id,
+					    rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 13/18] raw/ifpga/base: add secure support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 136 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 +++++++++++++-----
 4 files changed, 197 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..e597e47 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,62 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager"))
+			sysmgr_offset = offset;
+			break;
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +575,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 14/18] raw/ifpga/base: configure FEC mode

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 15/18] raw/ifpga/base: clean fme errors

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occurred.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 5d6d630..5905eac 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 01ff76a..95f079a 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1174,6 +1174,25 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1183,6 +1202,9 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 16/18] raw/ifpga/base: add new API get board info

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..9f0147d 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disaggregate;
+	u8 lightweight;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..799d67d 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweight = id.lightweiht;
+	fme->board_info.disaggregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweight:%s\n"
+			"disaggregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweight),
+			check_support(fme->board_info.disaggregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 17/18] raw/ifpga: add lightweight fpga image support

[Andy Pei]

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 95f079a..e87af66 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -835,6 +835,8 @@ static int set_surprise_link_check_aer(
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -861,22 +863,40 @@ static int set_surprise_link_check_aer(
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweight) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
-		uuid.b + 8, sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v8 18/18] raw/ifpga/base: add multiple cards support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, there is one MAX10 chip in each card, and
all of the sensors are connected to MAX10 chip. To support multiple
cards in one server, we introducing a sensor device list under
intel_max10_device instead of a global list. On the other hand, we
using seperate intel_max10_device instance for each opae_adatper.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c        |  40 ++++++++---
 drivers/raw/ifpga/base/opae_debug.c       |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c      |  14 ++--
 drivers/raw/ifpga/base/opae_hw_api.h      |  15 ++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 110 ++++++++++++++++--------------
 drivers/raw/ifpga/base/opae_intel_max10.h |  19 ++++--
 drivers/raw/ifpga/base/opae_spi.c         |   1 +
 drivers/raw/ifpga/base/opae_spi.h         |   2 +-
 drivers/raw/ifpga/ifpga_rawdev.c          |  14 ++--
 9 files changed, 134 insertions(+), 84 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 799d67d..c31a94c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -839,8 +839,13 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	struct ifpga_hw *hw;
 	u32 val;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
@@ -854,7 +859,10 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	fme->board_info.seu = id.seu;
 	fme->board_info.ptp = id.ptp;
 
-	dev_info(fme, "found: board: %s type: %s\n",
+	dev_info(fme, "found: PCI dev: %02x:%02x:%x board: %s type: %s\n",
+			hw->pci_data->bus,
+			hw->pci_data->devid,
+			hw->pci_data->function,
 			board_major_to_string(fme->board_info.major),
 			board_type_to_string(fme->board_info.type));
 
@@ -882,11 +890,11 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
-	if (max10_sys_read(MAX10_BUILD_VER, &val))
+	if (max10_sys_read(fme->max10_dev, MAX10_BUILD_VER, &val))
 		return -EINVAL;
 	fme->board_info.max10_version = val & 0xffffff;
 
-	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+	if (max10_sys_read(fme->max10_dev, NIOS2_FW_VERSION, &val))
 		return -EINVAL;
 	fme->board_info.nios_fw_version = val & 0xffffff;
 
@@ -897,12 +905,12 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	return 0;
 }
 
-static int spi_self_checking(void)
+static int spi_self_checking(struct intel_max10_device *dev)
 {
 	u32 val;
 	int ret;
 
-	ret = max10_sys_read(MAX10_TEST_REG, &val);
+	ret = max10_sys_read(dev, MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
@@ -937,10 +945,11 @@ static int fme_spi_init(struct ifpga_feature *feature)
 		goto spi_fail;
 	}
 
+
 	fme->max10_dev = max10;
 
 	/* SPI self test */
-	if (spi_self_checking()) {
+	if (spi_self_checking(max10)) {
 		ret = -EIO;
 		goto max10_fail;
 	}
@@ -1041,8 +1050,18 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 	struct ifpga_fme_hw *fme = (struct ifpga_fme_hw *)feature->parent;
 	struct altera_spi_device *spi_master;
 	struct intel_max10_device *max10;
+	struct ifpga_hw *hw;
+	struct opae_manager *mgr;
 	int ret = 0;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
+	mgr = hw->adapter->mgr;
+	if (!mgr)
+		return -ENODEV;
+
 	dev_info(fme, "FME SPI Master (NIOS) Init.\n");
 	dev_debug(fme, "FME SPI base addr %p.\n",
 			feature->addr);
@@ -1080,12 +1099,15 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 		goto release_dev;
 	}
 
+	max10->bus = hw->pci_data->bus;
+
 	fme_get_board_interface(fme);
 
 	fme->max10_dev = max10;
+	mgr->sensor_list = &max10->opae_sensor_list;
 
 	/* SPI self test */
-	if (spi_self_checking())
+	if (spi_self_checking(max10))
 		goto spi_fail;
 
 	return ret;
@@ -1344,7 +1366,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(dev, PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1372,7 +1394,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(sensor->value_reg, value)) {
+	if (max10_sys_read(dev, sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_debug.c b/drivers/raw/ifpga/base/opae_debug.c
index 88f2d5c..dad3ea3 100644
--- a/drivers/raw/ifpga/base/opae_debug.c
+++ b/drivers/raw/ifpga/base/opae_debug.c
@@ -59,6 +59,9 @@ static void opae_adapter_data_dump(void *data)
 		opae_log("OPAE Adapter Type = PCI\n");
 		opae_log("PCI Device ID: 0x%04x\n", d_pci->device_id);
 		opae_log("PCI Vendor ID: 0x%04x\n", d_pci->vendor_id);
+		opae_log("PCI bus: 0x%04x\n", d_pci->bus);
+		opae_log("PCI devid: 0x%04x\n", d_pci->devid);
+		opae_log("PCI function: 0x%04x\n", d_pci->function);
 
 		for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 			r = &d_pci->region[i];
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 1ccc967..c969dfe 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -583,11 +583,12 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_id(unsigned int id)
+opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (sensor->id == id)
 			return sensor;
 
@@ -601,11 +602,12 @@ struct opae_sensor_info *
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_name(const char *name)
+opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (!strcmp(sensor->name, name))
 			return sensor;
 
@@ -630,7 +632,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_name(name);
+	sensor = opae_mgr_get_sensor_by_name(mgr, name);
 	if (!sensor)
 		return -ENODEV;
 
@@ -658,7 +660,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_id(id);
+	sensor = opae_mgr_get_sensor_by_id(mgr, id);
 	if (!sensor)
 		return -ENODEV;
 
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index b78fbd5..cdf369f 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -40,6 +40,7 @@ struct opae_manager {
 	struct opae_adapter *adapter;
 	struct opae_manager_ops *ops;
 	struct opae_manager_networking_ops *network_ops;
+	struct opae_sensor_list *sensor_list;
 	void *data;
 };
 
@@ -75,9 +76,8 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
-extern struct opae_sensor_list opae_sensor_list;
-#define opae_mgr_for_each_sensor(sensor) \
-	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+#define opae_mgr_for_each_sensor(mgr, sensor) \
+	TAILQ_FOREACH(sensor, mgr->sensor_list, node)
 
 /* OPAE Manager APIs */
 struct opae_manager *
@@ -88,8 +88,10 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
-struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
-struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id);
 int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
 		const char *name, unsigned int *value);
 int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
@@ -241,6 +243,9 @@ struct opae_adapter_data_pci {
 	enum opae_adapter_type type;
 	u16 device_id;
 	u16 vendor_id;
+	u16 bus; /*Device bus for PCI */
+	u16 devid; /* Device ID */
+	u16 function; /* Device function */
 	struct opae_reg_region region[PCI_MAX_RESOURCE];
 	int vfio_dev_fd;  /* VFIO device file descriptor */
 };
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index e597e47..680a580 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -5,45 +5,50 @@
 #include "opae_intel_max10.h"
 #include <libfdt.h>
 
-static struct intel_max10_device *g_max10;
-
-struct opae_sensor_list opae_sensor_list =
-	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
-
-int max10_reg_read(unsigned int reg, unsigned int *val)
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_read(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x\n", __func__, dev->bus, reg);
+
+	return spi_transaction_read(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)val);
 }
 
-int max10_reg_write(unsigned int reg, unsigned int val)
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val)
 {
 	unsigned int tmp = val;
 
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_write(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x, val:0x%x\n", __func__,
+			dev->bus, reg, val);
+
+	return spi_transaction_write(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)&tmp);
 }
 
-int max10_sys_read(unsigned int offset, unsigned int *val)
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_read(g_max10->base + offset, val);
+
+	return max10_reg_read(dev, dev->base + offset, val);
 }
 
-int max10_sys_write(unsigned int offset, unsigned int val)
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_write(g_max10->base + offset, val);
+	return max10_reg_write(dev, dev->base + offset, val);
 }
 
 static struct max10_compatible_id max10_id_table[] = {
@@ -86,8 +91,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
-static int altera_nor_flash_read(u32 offset,
-		void *buffer, u32 len)
+static int altera_nor_flash_read(struct intel_max10_device *dev,
+	u32 offset, void *buffer, u32 len)
 {
 	int word_len;
 	int i;
@@ -95,13 +100,13 @@ static int altera_nor_flash_read(u32 offset,
 	unsigned int value;
 	int ret;
 
-	if (!buffer || len <= 0)
+	if (!dev || !buffer || len <= 0)
 		return -ENODEV;
 
 	word_len = len/4;
 
 	for (i = 0; i < word_len; i++) {
-		ret = max10_reg_read(offset + i*4,
+		ret = max10_reg_read(dev, offset + i*4,
 				&value);
 		if (ret)
 			return -EBUSY;
@@ -112,12 +117,12 @@ static int altera_nor_flash_read(u32 offset,
 	return 0;
 }
 
-static int enable_nor_flash(bool on)
+static int enable_nor_flash(struct intel_max10_device *dev, bool on)
 {
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_sys_read(RSU_REG, &val);
+	ret = max10_sys_read(dev, RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -128,7 +133,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_sys_write(RSU_REG, val);
+	return max10_sys_write(dev, RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -140,7 +145,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_sys_read(DT_AVAIL_REG, &val);
+	ret = max10_sys_read(max10, DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -151,19 +156,19 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
+	ret = max10_sys_read(max10, DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
 	}
 
-	ret = enable_nor_flash(true);
+	ret = enable_nor_flash(max10, true);
 	if (ret) {
 		dev_err(max10 "fail to enable flash\n");
 		return ret;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	ret = altera_nor_flash_read(max10, dt_addr, &hdr, sizeof(hdr));
 	if (ret) {
 		dev_err(max10 "read fdt header fail\n");
 		goto done;
@@ -188,7 +193,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		goto done;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	ret = altera_nor_flash_read(max10, dt_addr, fdt_root, dt_size);
 	if (ret) {
 		dev_err(max10 "cannot read device table\n");
 		goto done;
@@ -207,7 +212,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	max10->fdt_root = fdt_root;
 
 done:
-	ret = enable_nor_flash(false);
+	ret = enable_nor_flash(max10, false);
 
 	if (ret && fdt_root)
 		opae_free(fdt_root);
@@ -298,12 +303,12 @@ static int fdt_get_named_reg(const void *fdt, int node, const char *name,
 	return fdt_get_reg(fdt, node, idx, start, size);
 }
 
-static void max10_sensor_uinit(void)
+static void max10_sensor_uinit(struct intel_max10_device *dev)
 {
 	struct opae_sensor_info *info;
 
-	TAILQ_FOREACH(info, &opae_sensor_list, node) {
-		TAILQ_REMOVE(&opae_sensor_list, info, node);
+	TAILQ_FOREACH(info, &dev->opae_sensor_list, node) {
+		TAILQ_REMOVE(&dev->opae_sensor_list, info, node);
 		opae_free(info);
 	}
 }
@@ -313,8 +318,8 @@ static bool sensor_reg_valid(struct sensor_reg *reg)
 	return !!reg->size;
 }
 
-static int max10_add_sensor(struct raw_sensor_info *info,
-		struct opae_sensor_info *sensor)
+static int max10_add_sensor(struct intel_max10_device *dev,
+	struct raw_sensor_info *info, struct opae_sensor_info *sensor)
 {
 	int i;
 	int ret = 0;
@@ -332,12 +337,15 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_sys_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(dev, info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
-		if (val == 0xdeadbeef)
+		if (val == 0xdeadbeef) {
+			dev_debug(dev, "%s: sensor:%s invalid 0x%x at:%d\n",
+				__func__, sensor->name, val, i);
 			continue;
+		}
 
 		val *= info->multiplier;
 
@@ -458,7 +466,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
 		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
 
-		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+		dev_debug(dev, "found sensor from DTB: %s: %s: %u: %u\n",
 				raw->name, raw->type,
 				raw->id, raw->multiplier);
 
@@ -473,15 +481,16 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 			goto free_sensor;
 		}
 
-		if (max10_add_sensor(raw, sensor)) {
+		if (max10_add_sensor(dev, raw, sensor)) {
 			ret = -EINVAL;
 			opae_free(sensor);
 			goto free_sensor;
 		}
 
-		if (sensor->flags & OPAE_SENSOR_VALID)
-			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
-		else
+		if (sensor->flags & OPAE_SENSOR_VALID) {
+			TAILQ_INSERT_TAIL(&dev->opae_sensor_list, sensor, node);
+			dev_info(dev, "found valid sensor: %s\n", sensor->name);
+		} else
 			opae_free(sensor);
 
 		opae_free(raw);
@@ -492,7 +501,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 free_sensor:
 	if (raw)
 		opae_free(raw);
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 	return ret;
 }
 
@@ -500,7 +509,7 @@ static int check_max10_version(struct intel_max10_device *dev)
 {
 	unsigned int v;
 
-	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+	if (!max10_reg_read(dev, MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
 				&v)) {
 		if (v != 0xffffffff) {
 			dev_info(dev, "secure MAX10 detected\n");
@@ -564,6 +573,8 @@ struct intel_max10_device *
 	if (!dev)
 		return NULL;
 
+	TAILQ_INIT(&dev->opae_sensor_list);
+
 	dev->spi_master = spi;
 
 	dev->spi_tran_dev = spi_transaction_init(spi, chipselect);
@@ -572,9 +583,6 @@ struct intel_max10_device *
 		goto free_dev;
 	}
 
-	/* set the max10 device firstly */
-	g_max10 = dev;
-
 	/* check the max10 version */
 	ret = check_max10_version(dev);
 	if (ret) {
@@ -600,7 +608,7 @@ struct intel_max10_device *
 	}
 
 	/* read FPGA loading information */
-	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
+	ret = max10_sys_read(dev, FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
 		goto release_max10_hw;
@@ -610,14 +618,13 @@ struct intel_max10_device *
 	return dev;
 
 release_max10_hw:
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return NULL;
@@ -628,7 +635,7 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
@@ -636,7 +643,6 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index e632941..123cdc4 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -26,6 +26,9 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SECURE		BIT(6)
 #define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
@@ -33,6 +36,8 @@ struct intel_max10_device {
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
 	unsigned int base; /* max10 base address */
+	u16 bus;
+	struct opae_sensor_list opae_sensor_list;
 };
 
 /* retimer speed */
@@ -136,17 +141,19 @@ struct opae_retimer_status {
 
 #define DFT_MAX_SIZE		0x7e0000
 
-int max10_reg_read(unsigned int reg, unsigned int *val);
-int max10_reg_write(unsigned int reg, unsigned int val);
-int max10_sys_read(unsigned int offset, unsigned int *val);
-int max10_sys_write(unsigned int offset, unsigned int val);
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val);
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val);
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val);
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
-/** List of opae sensors */
-TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
 
 #define SENSOR_REG_VALUE 0x0
 #define SENSOR_REG_HIGH_WARN 0x1
diff --git a/drivers/raw/ifpga/base/opae_spi.c b/drivers/raw/ifpga/base/opae_spi.c
index cc52782..2a5e929 100644
--- a/drivers/raw/ifpga/base/opae_spi.c
+++ b/drivers/raw/ifpga/base/opae_spi.c
@@ -194,6 +194,7 @@ static int spi_txrx(struct altera_spi_device *dev)
 				return -EIO;
 			if (status & ALTERA_SPI_STATUS_RRDY_MSK)
 				break;
+			udelay(1);
 			if (retry++ > SPI_MAX_RETRY) {
 				dev_err(dev, "%s, read timeout\n", __func__);
 				return -EBUSY;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index 6355deb..db56877 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -38,7 +38,7 @@
 #define SPI_WRITE 0x20
 #define WRITE_DATA_MASK GENMASK_ULL(31, 0)
 
-#define SPI_MAX_RETRY 100000
+#define SPI_MAX_RETRY 1000000
 
 #define TYPE_SPI 0
 #define TYPE_NIOS_SPI 1
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index e87af66..7253644 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -360,7 +360,7 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 	if (!mgr)
 		return -ENODEV;
 
-	opae_mgr_for_each_sensor(sensor) {
+	opae_mgr_for_each_sensor(mgr, sensor) {
 		if (!(sensor->flags & OPAE_SENSOR_VALID))
 			goto fail;
 
@@ -369,8 +369,8 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 			goto fail;
 
 		if (value == 0xdeadbeef) {
-			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
-					sensor->name, value);
+			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value %x\n",
+					raw_dev->dev_id, sensor->name, value);
 			continue;
 		}
 
@@ -393,8 +393,9 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 		/* monitor 12V AUX sensor */
 		if (!strcmp(sensor->name, "12V AUX Voltage")) {
 			if (value < AUX_VOLTAGE_WARN) {
-				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
-						sensor->name, value);
+				IFPGA_RAWDEV_PMD_INFO(
+					"%s reach theshold %d mV\n",
+					sensor->name, value);
 				*gsd_start = true;
 				break;
 			}
@@ -1433,6 +1434,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->bus = pci_dev->addr.bus;
+	data->devid = pci_dev->addr.devid;
+	data->function = pci_dev->addr.function;
 	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 02/17] raw/ifpga/base: add irq support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME global error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

To build this feature, CONFIG_RTE_EAL_VFIO should be enabled.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 config/common_base                         |  2 +-
 config/common_linux                        |  6 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 59 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 19 ++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 18 +++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 19 ++++++++++
 6 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/config/common_base b/config/common_base
index 8ef75c2..1bbe012 100644
--- a/config/common_base
+++ b/config/common_base
@@ -768,7 +768,7 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
 #
 # Compile PMD for Intel FPGA raw device
 #
-CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=n
 
 #
 # Compile PMD for Intel IOAT raw device
diff --git a/config/common_linux b/config/common_linux
index 6e25255..0dc2d82 100644
--- a/config/common_linux
+++ b/config/common_linux
@@ -63,3 +63,9 @@ CONFIG_RTE_LIBRTE_ENETC_PMD=y
 # HINIC PMD driver
 #
 CONFIG_RTE_LIBRTE_HINIC_PMD=y
+
+#
+# Compile PMD for Intel FPGA raw device
+# To compile, CONFIG_RTE_EAL_VFIO should be enabled.
+#
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..0f852a7 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,61 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+#define FPGA_MAX_MSIX_VEC_COUNT	128
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MAX_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	/* 0 < irq_set->count < FPGA_MAX_MSIX_VEC_COUNT */
+	irq_set->count = count ?
+		(count > FPGA_MAX_MSIX_VEC_COUNT ?
+		 FPGA_MAX_MSIX_VEC_COUNT : count) : 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..2978c79 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,28 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&fme->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..c0aaf01 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,27 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..189f762 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,28 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ))
+		return -ENODEV;
+
+	spinlock_lock(&port->lock);
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 03/17] raw/ifpga/base: clear pending bit

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 2978c79..be041ec 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 04/17] raw/ifpga/base: add SEU error support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 ++++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 31 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index be041ec..5d6d630 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,33 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr(struct ifpga_fme_hw *fme,
+		u64 *val, bool high)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+				FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	if (high)
+		*val = readq(&fme_err->seu_emr_h);
+	else
+		*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +297,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr(fme, &prop->data, 0);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr(fme, &prop->data, 1);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 05/17] raw/ifpga/base: add device tree support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 mk/rte.app.mk                             |   2 +-
 3 files changed, 194 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index ba5c39e..2acf9c0 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -319,7 +319,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v7 05/17] raw/ifpga/base: add device tree support

[Ye Xiaolong]

This patch introduces the dependency on libfdt, so we need to disable
ipn3ke driver by default in config/common_base, otherwise it would cause
a build error.

On 09/26, Andy Pei wrote:
>From: Tianfei zhang <tianfei.zhang@intel.com>
>
>In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
>to manage the board configuration, like sensors, flash controller,
>QSFP, powers. And this is a SPI bus connected between A10 FPGA and
>MAX10, we can access the MAX10 registers over this SPI bus.
>
>In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
>core temperature, Board temperature, board current, voltage and so on.
>
>We use DTB (Device tree table) to describe it. This DTB file is store
>in nor flash partition, which will flashed in Factory when the boards
>delivery to customers. And the same time, the customers can easy to
>customizate the BMC configuration like change the sensors.
>
>Add device tree support by using libfdt library in Linux distribution.
>The end-user should pre-install the libfdt and libfdt-devel package
>before use DPDK on PAC N3000 Card.
>
>For Centos 7.x: sudo yum install libfdt libfdt-devel
>For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

We'll need to add this info to document.

Thanks,
Xiaolong

Re: [dpdk-dev] [PATCH v7 05/17] raw/ifpga/base: add device tree support

[Pei, Andy]

Hi Xiaolong

I will do that in v8.

-----Original Message-----
From: Ye, Xiaolong 
Sent: Friday, September 27, 2019 6:33 PM
To: Pei, Andy <andy.pei@intel.com>
Cc: dev@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>; Yigit, Ferruh <ferruh.yigit@intel.com>
Subject: Re: [PATCH v7 05/17] raw/ifpga/base: add device tree support

This patch introduces the dependency on libfdt, so we need to disable ipn3ke driver by default in config/common_base, otherwise it would cause a build error.

On 09/26, Andy Pei wrote:
>From: Tianfei zhang <tianfei.zhang@intel.com>
>
>In PAC N3000 card, this is a BMC chip which using MAX10 FPGA to manage 
>the board configuration, like sensors, flash controller, QSFP, powers. 
>And this is a SPI bus connected between A10 FPGA and MAX10, we can 
>access the MAX10 registers over this SPI bus.
>
>In BMC, there are about 19 sensors in MAX10 chip, including the FPGA 
>core temperature, Board temperature, board current, voltage and so on.
>
>We use DTB (Device tree table) to describe it. This DTB file is store 
>in nor flash partition, which will flashed in Factory when the boards 
>delivery to customers. And the same time, the customers can easy to 
>customizate the BMC configuration like change the sensors.
>
>Add device tree support by using libfdt library in Linux distribution.
>The end-user should pre-install the libfdt and libfdt-devel package 
>before use DPDK on PAC N3000 Card.
>
>For Centos 7.x: sudo yum install libfdt libfdt-devel For Ubuntu 18.04: 
>sudo apt install libfdt-dev libfdt1

We'll need to add this info to document.

Thanks,
Xiaolong

[dpdk-dev] [PATCH v7 06/17] raw/ifpga/base: align the send buffer for SPI

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 07/17] raw/ifpga/base: add sensor support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 08/17] raw/ifpga/base: introducing sensor APIs

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 09/17] raw/ifpga/base: update SEU register definition

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 10/17] raw/ifpga: add SEU error handler

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 247 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 247 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index fef89e6..bdd7a69 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -28,6 +28,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -606,6 +608,238 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int
+fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int
+ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -653,6 +887,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -678,6 +913,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -697,6 +936,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -721,6 +961,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 11/17] raw/ifpga: add PCIe BDF devices tree scan

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 567 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 571 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index bdd7a69..b4e7ec1 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_eal_memconfig.h>
@@ -18,7 +20,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -26,6 +28,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -38,6 +41,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -65,6 +74,496 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+
+static int
+set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev);
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+		int start, int cap);
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int
+ifpga_pci_find_next_ext_capability(unsigned int fd,
+		int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int
+ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int
+ifpga_get_dev_vendor_id(const char *bdf, uint32_t *dev_id,
+		uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int
+ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev, const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev, bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor is invalid value %x\n",
+					value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+			!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int
+set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev);
+				if (ret == 1)
+					gsd_enable = 1;
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100000 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -373,8 +872,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+		sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -845,6 +1345,7 @@
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -858,7 +1359,7 @@
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -872,6 +1373,14 @@
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -990,6 +1499,7 @@
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1022,12 +1532,32 @@
 };
 
 static int
+ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+		const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
+static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1041,14 +1571,15 @@
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+			&ifpga_rawdev_get_string_arg,
+			&name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
-				     IFPGA_ARG_NAME);
+				IFPGA_ARG_NAME);
 			goto end;
 		}
 	} else {
 		IFPGA_RAWDEV_PMD_ERR("arg %s is mandatory for ifpga bus",
-			  IFPGA_ARG_NAME);
+			IFPGA_ARG_NAME);
 		goto end;
 	}
 
@@ -1063,16 +1594,28 @@
 		}
 	} else {
 		IFPGA_RAWDEV_PMD_ERR("arg %s is mandatory for ifpga bus",
-			  IFPGA_ARG_PORT);
+			IFPGA_ARG_PORT);
 		goto end;
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
-	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
-	port, name);
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s", port, name);
 
-	ret = rte_eal_hotplug_add(RTE_STR(IFPGA_BUS_NAME),
-			dev_name, devargs->args);
+	ret = rte_eal_hotplug_add(RTE_STR(IFPGA_BUS_NAME), dev_name,
+			devargs->args);
 end:
 	if (kvlist)
 		rte_kvargs_free(kvlist);
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 12/17] net/ipn3ke: remove configuration for i40e port bonding

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/ipn3ke/Makefile             |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c      | 291 +++++---------------------------
 drivers/net/ipn3ke/ipn3ke_representor.c |   8 +-
 3 files changed, 46 insertions(+), 255 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..2c65e49 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index c226d63..86ed087 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -241,7 +242,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -259,6 +261,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -323,9 +331,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -376,7 +381,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -403,7 +412,14 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+	if (!ifpga_dev) {
+		IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+		return -ENODEV;
+	}
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -415,6 +431,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -422,6 +454,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -467,254 +500,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index 8300cc3..9a17f4b 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2906,8 +2907,11 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		rte_pmd_i40e_set_switch_dev(rpst->i40e_pf_eth_port_id,
+					    rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 13/17] raw/ifpga/base: add secure support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 136 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 +++++++++++++-----
 4 files changed, 197 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..e597e47 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,62 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager"))
+			sysmgr_offset = offset;
+			break;
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +575,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 14/17] raw/ifpga/base: configure FEC mode

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 15/17] raw/ifpga/base: clean fme errors

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occurred.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 5d6d630..5905eac 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index b4e7ec1..5fa40f6 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1178,6 +1178,25 @@ struct ifpga_rawdev *
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1187,6 +1206,9 @@ struct ifpga_rawdev *
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 16/17] raw/ifpga/base: add new API get board info

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..e529f54 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disagregate;
+	u8 lightweiht;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..1ebafae 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweiht = id.lightweiht;
+	fme->board_info.disagregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweiht:%s\n"
+			"disagregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweiht),
+			check_support(fme->board_info.disagregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v7 17/17] raw/ifpga: add lightweight fpga image support

[Andy Pei]

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 5fa40f6..59f91b7 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -838,6 +838,8 @@ struct ifpga_rawdev *
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -864,22 +866,40 @@ struct ifpga_rawdev *
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweiht) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
-		sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 02/17] raw/ifpga/base: add irq support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME globle error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 60 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 22 +++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 20 ++++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 21 +++++++++++
 4 files changed, 123 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..f0fb242 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,62 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+/*
+ * FIXME: we should get msix vec count during pci enumeration instead of
+ * below hardcode value.
+ */
+#define FPGA_MSIX_VEC_COUNT	20
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	irq_set->count = count;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..068f52c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,31 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set =
+			(struct fpga_fme_err_irq_set *)irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	spinlock_lock(&fme->lock);
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ)) {
+		spinlock_unlock(&fme->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..56b04a6 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,29 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	spinlock_lock(&port->lock);
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ)) {
+		spinlock_unlock(&port->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..8aef7d7 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,30 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	spinlock_lock(&port->lock);
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ)) {
+		spinlock_unlock(&port->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v6 02/17] raw/ifpga/base: add irq support

[Ye Xiaolong]

On 09/19, Andy Pei wrote:
>From: Tianfei zhang <tianfei.zhang@intel.com>
>
>Add irq support for ifpga FME globle error, port error and uint unit.

s/globle/global

>We implmented this feature by vfio interrupt mechanism.
>
>Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
>Signed-off-by: Andy Pei <andy.pei@intel.com>
>---
> drivers/raw/ifpga/base/ifpga_feature_dev.c | 60 ++++++++++++++++++++++++++++++
> drivers/raw/ifpga/base/ifpga_fme_error.c   | 22 +++++++++++
> drivers/raw/ifpga/base/ifpga_port.c        | 20 ++++++++++
> drivers/raw/ifpga/base/ifpga_port_error.c  | 21 +++++++++++
> 4 files changed, 123 insertions(+)
>
>diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
>index 63c8bcc..f0fb242 100644
>--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
>+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
>@@ -3,6 +3,7 @@
>  */
> 
> #include <sys/ioctl.h>
>+#include <rte_vfio.h>
> 
> #include "ifpga_feature_dev.h"
> 
>@@ -331,3 +332,62 @@ int port_hw_init(struct ifpga_port_hw *port)
> 	port_hw_uinit(port);
> 	return ret;
> }
>+
>+/*
>+ * FIXME: we should get msix vec count during pci enumeration instead of
>+ * below hardcode value.
>+ */
>+#define FPGA_MSIX_VEC_COUNT	20

So what is preventing us from getting msix vec count from pci enumeration?

>+/* irq set buffer length for interrupt */
>+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
>+				sizeof(int) * FPGA_MSIX_VEC_COUNT)
>+
>+/* only support msix for now*/
>+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
>+				  unsigned int count, s32 *fds)

DPDK convention is put the function return type in a separate line.

>+{
>+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
>+	struct vfio_irq_set *irq_set;
>+	int len, ret;
>+	int *fd_ptr;
>+
>+	len = sizeof(irq_set_buf);
>+
>+	irq_set = (struct vfio_irq_set *)irq_set_buf;
>+	irq_set->argsz = len;
>+	irq_set->count = count;
>+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
>+				VFIO_IRQ_SET_ACTION_TRIGGER;
>+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
>+	irq_set->start = vec_start;
>+
>+	fd_ptr = (int *)&irq_set->data;
>+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
>+
>+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
>+	if (ret)
>+		printf("Error enabling MSI-X interrupts\n");
>+
>+	return ret;
>+}
>+
>+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
>+			unsigned int count, s32 *fds)

Ditto.

>+{
>+	struct feature_irq_ctx *ctx = feature->ctx;
>+	unsigned int i;
>+	int ret;
>+
>+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
>+		return -EINVAL;
>+
>+	/* assume that each feature has continuous vector space in msix*/
>+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
>+				     ctx[start].idx, count, fds);
>+	if (!ret) {
>+		for (i = 0; i < count; i++)
>+			ctx[i].eventfd = fds[i];
>+	}
>+
>+	return ret;
>+}
>diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
>index 3794564..068f52c 100644
>--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
>+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
>@@ -373,9 +373,31 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
> 	return -ENOENT;
> }
> 
>+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)

Ditto.

>+{
>+	struct fpga_fme_err_irq_set *err_irq_set =
>+			(struct fpga_fme_err_irq_set *)irq_set;

Cast is not needed for void *.

>+	struct ifpga_fme_hw *fme;
>+	int ret;
>+
>+	fme = (struct ifpga_fme_hw *)feature->parent;

Ditto.

>+
>+	spinlock_lock(&fme->lock);
>+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ)) {
>+		spinlock_unlock(&fme->lock);
>+		return -ENODEV;
>+	}
>+
>+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
>+	spinlock_unlock(&fme->lock);
>+
>+	return ret;
>+}
>+
> struct ifpga_feature_ops fme_global_err_ops = {
> 	.init = fme_global_error_init,
> 	.uinit = fme_global_error_uinit,
> 	.get_prop = fme_global_error_get_prop,
> 	.set_prop = fme_global_error_set_prop,
>+	.set_irq = fme_global_err_set_irq,
> };
>diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
>index 6c41164..56b04a6 100644
>--- a/drivers/raw/ifpga/base/ifpga_port.c
>+++ b/drivers/raw/ifpga/base/ifpga_port.c
>@@ -384,9 +384,29 @@ static void port_uint_uinit(struct ifpga_feature *feature)
> 	dev_info(NULL, "PORT UINT UInit.\n");
> }
> 
>+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)

Ditto.

>+{
>+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
>+	struct ifpga_port_hw *port = feature->parent;
>+	int ret;
>+
>+	spinlock_lock(&port->lock);
>+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ)) {
>+		spinlock_unlock(&port->lock);
>+		return -ENODEV;
>+	}
>+
>+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
>+				  uafu_irq_set->count, uafu_irq_set->evtfds);
>+	spinlock_unlock(&port->lock);
>+
>+	return ret;
>+}
>+
> struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
> 	.init = port_uint_init,
> 	.uinit = port_uint_uinit,
>+	.set_irq = port_uint_set_irq,
> };
> 
> static int port_afu_init(struct ifpga_feature *feature)
>diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
>index 138284e..8aef7d7 100644
>--- a/drivers/raw/ifpga/base/ifpga_port_error.c
>+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
>@@ -136,9 +136,30 @@ static int port_error_set_prop(struct ifpga_feature *feature,
> 	return -ENOENT;
> }
> 
>+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)

Ditto.

>+{
>+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
>+	struct ifpga_port_hw *port;
>+	int ret;
>+
>+	port = feature->parent;
>+
>+	spinlock_lock(&port->lock);
>+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ)) {
>+		spinlock_unlock(&port->lock);
>+		return -ENODEV;
>+	}
>+
>+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
>+	spinlock_unlock(&port->lock);
>+
>+	return ret;
>+}
>+

Above 3 new functions have a lot of similarity, better to extract out common
function to reduce duplication.

Thanks,
Xiaolong

> struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
> 	.init = port_error_init,
> 	.uinit = port_error_uinit,
> 	.get_prop = port_error_get_prop,
> 	.set_prop = port_error_set_prop,
>+	.set_irq = port_error_set_irq,
> };
>-- 
>1.8.3.1
>

Re: [dpdk-dev] [PATCH v6 02/17] raw/ifpga/base: add irq support

[Ye Xiaolong]

On 09/25, Ye Xiaolong wrote:
>>+/* only support msix for now*/
>>+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
>>+				  unsigned int count, s32 *fds)
>
>DPDK convention is put the function return type in a separate line.
>

Just noticed that this is a share code update, then this rule doesn't need to
be followed.

Thanks,
Xiaolong

Re: [dpdk-dev] [PATCH v6 02/17] raw/ifpga/base: add irq support

[Zhang, Tianfei]

> -----Original Message-----
> From: Ye, Xiaolong
> Sent: Wednesday, September 25, 2019 12:02 AM
> To: Pei, Andy <andy.pei@intel.com>
> Cc: dev@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> <tianfei.zhang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Lomartire,
> David <david.lomartire@intel.com>; Yigit, Ferruh <ferruh.yigit@intel.com>
> Subject: Re: [PATCH v6 02/17] raw/ifpga/base: add irq support
> 
> On 09/19, Andy Pei wrote:
> >From: Tianfei zhang <tianfei.zhang@intel.com>
> >
> >Add irq support for ifpga FME globle error, port error and uint unit.
> 
> s/globle/global

Thanks, will fix in next version.

> 
> >We implmented this feature by vfio interrupt mechanism.
> >
> >Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
> >Signed-off-by: Andy Pei <andy.pei@intel.com>
> >---
> > drivers/raw/ifpga/base/ifpga_feature_dev.c | 60
> ++++++++++++++++++++++++++++++
> > drivers/raw/ifpga/base/ifpga_fme_error.c   | 22 +++++++++++
> > drivers/raw/ifpga/base/ifpga_port.c        | 20 ++++++++++
> > drivers/raw/ifpga/base/ifpga_port_error.c  | 21 +++++++++++
> > 4 files changed, 123 insertions(+)
> >
> >diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c
> >b/drivers/raw/ifpga/base/ifpga_feature_dev.c
> >index 63c8bcc..f0fb242 100644
> >--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
> >+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
> >@@ -3,6 +3,7 @@
> >  */
> >
> > #include <sys/ioctl.h>
> >+#include <rte_vfio.h>
> >
> > #include "ifpga_feature_dev.h"
> >
> >@@ -331,3 +332,62 @@ int port_hw_init(struct ifpga_port_hw *port)
> > 	port_hw_uinit(port);
> > 	return ret;
> > }
> >+
> >+/*
> >+ * FIXME: we should get msix vec count during pci enumeration instead
> >+of
> >+ * below hardcode value.
> >+ */
> >+#define FPGA_MSIX_VEC_COUNT	20
> 
> So what is preventing us from getting msix vec count from pci enumeration?
> 
> >+/* irq set buffer length for interrupt */ #define MSIX_IRQ_SET_BUF_LEN
> >+(sizeof(struct vfio_irq_set) + \
> >+				sizeof(int) * FPGA_MSIX_VEC_COUNT)
> >+
> >+/* only support msix for now*/
> >+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
> >+				  unsigned int count, s32 *fds)
> 
> DPDK convention is put the function return type in a separate line.
> 
> >+{
> >+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
> >+	struct vfio_irq_set *irq_set;
> >+	int len, ret;
> >+	int *fd_ptr;
> >+
> >+	len = sizeof(irq_set_buf);
> >+
> >+	irq_set = (struct vfio_irq_set *)irq_set_buf;
> >+	irq_set->argsz = len;
> >+	irq_set->count = count;
> >+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
> >+				VFIO_IRQ_SET_ACTION_TRIGGER;
> >+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
> >+	irq_set->start = vec_start;
> >+
> >+	fd_ptr = (int *)&irq_set->data;
> >+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
> >+
> >+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
> >+	if (ret)
> >+		printf("Error enabling MSI-X interrupts\n");
> >+
> >+	return ret;
> >+}
> >+
> >+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
> >+			unsigned int count, s32 *fds)
> 
> Ditto.
> 
> >+{
> >+	struct feature_irq_ctx *ctx = feature->ctx;
> >+	unsigned int i;
> >+	int ret;
> >+
> >+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
> >+		return -EINVAL;
> >+
> >+	/* assume that each feature has continuous vector space in msix*/
> >+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
> >+				     ctx[start].idx, count, fds);
> >+	if (!ret) {
> >+		for (i = 0; i < count; i++)
> >+			ctx[i].eventfd = fds[i];
> >+	}
> >+
> >+	return ret;
> >+}
> >diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c
> >b/drivers/raw/ifpga/base/ifpga_fme_error.c
> >index 3794564..068f52c 100644
> >--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
> >+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
> >@@ -373,9 +373,31 @@ static int fme_global_error_set_prop(struct
> ifpga_feature *feature,
> > 	return -ENOENT;
> > }
> >
> >+static int fme_global_err_set_irq(struct ifpga_feature *feature, void
> >+*irq_set)
> 
> Ditto.
> 
> >+{
> >+	struct fpga_fme_err_irq_set *err_irq_set =
> >+			(struct fpga_fme_err_irq_set *)irq_set;
> 
> Cast is not needed for void *.
Will fix in next version.

> 
> >+	struct ifpga_fme_hw *fme;
> >+	int ret;
> >+
> >+	fme = (struct ifpga_fme_hw *)feature->parent;
> 
> Ditto.
> 
> >+
> >+	spinlock_lock(&fme->lock);
> >+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ)) {
> >+		spinlock_unlock(&fme->lock);
> >+		return -ENODEV;
> >+	}
> >+
> >+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
> >+	spinlock_unlock(&fme->lock);
> >+
> >+	return ret;
> >+}
> >+
> > struct ifpga_feature_ops fme_global_err_ops = {
> > 	.init = fme_global_error_init,
> > 	.uinit = fme_global_error_uinit,
> > 	.get_prop = fme_global_error_get_prop,
> > 	.set_prop = fme_global_error_set_prop,
> >+	.set_irq = fme_global_err_set_irq,
> > };
> >diff --git a/drivers/raw/ifpga/base/ifpga_port.c
> >b/drivers/raw/ifpga/base/ifpga_port.c
> >index 6c41164..56b04a6 100644
> >--- a/drivers/raw/ifpga/base/ifpga_port.c
> >+++ b/drivers/raw/ifpga/base/ifpga_port.c
> >@@ -384,9 +384,29 @@ static void port_uint_uinit(struct ifpga_feature
> *feature)
> > 	dev_info(NULL, "PORT UINT UInit.\n");  }
> >
> >+static int port_uint_set_irq(struct ifpga_feature *feature, void
> >+*irq_set)
> 
> Ditto.
> 
> >+{
> >+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
> >+	struct ifpga_port_hw *port = feature->parent;
> >+	int ret;
> >+
> >+	spinlock_lock(&port->lock);
> >+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ)) {
> >+		spinlock_unlock(&port->lock);
> >+		return -ENODEV;
> >+	}
> >+
> >+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
> >+				  uafu_irq_set->count, uafu_irq_set->evtfds);
> >+	spinlock_unlock(&port->lock);
> >+
> >+	return ret;
> >+}
> >+
> > struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
> > 	.init = port_uint_init,
> > 	.uinit = port_uint_uinit,
> >+	.set_irq = port_uint_set_irq,
> > };
> >
> > static int port_afu_init(struct ifpga_feature *feature) diff --git
> >a/drivers/raw/ifpga/base/ifpga_port_error.c
> >b/drivers/raw/ifpga/base/ifpga_port_error.c
> >index 138284e..8aef7d7 100644
> >--- a/drivers/raw/ifpga/base/ifpga_port_error.c
> >+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
> >@@ -136,9 +136,30 @@ static int port_error_set_prop(struct ifpga_feature
> *feature,
> > 	return -ENOENT;
> > }
> >
> >+static int port_error_set_irq(struct ifpga_feature *feature, void
> >+*irq_set)
> 
> Ditto.
> 
> >+{
> >+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
> >+	struct ifpga_port_hw *port;
> >+	int ret;
> >+
> >+	port = feature->parent;
> >+
> >+	spinlock_lock(&port->lock);
> >+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ)) {
> >+		spinlock_unlock(&port->lock);
> >+		return -ENODEV;
> >+	}
> >+
> >+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
> >+	spinlock_unlock(&port->lock);
> >+
> >+	return ret;
> >+}
> >+
> 
> Above 3 new functions have a lot of similarity, better to extract out common
> function to reduce duplication.

Good suggestion, will fix in next version.

> 
> Thanks,
> Xiaolong
> 
> > struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
> > 	.init = port_error_init,
> > 	.uinit = port_error_uinit,
> > 	.get_prop = port_error_get_prop,
> > 	.set_prop = port_error_set_prop,
> >+	.set_irq = port_error_set_irq,
> > };
> >--
> >1.8.3.1
> >

[dpdk-dev] [PATCH v6 03/17] raw/ifpga/base: clear pending bit

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 068f52c..a6d3dab 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 04/17] raw/ifpga/base: add SEU error support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 +++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 43 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index a6d3dab..c9bac15 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,45 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr_low(struct ifpga_fme_hw *fme,
+		u64 *val)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+						  FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
+static int fme_err_get_seu_emr_high(struct ifpga_fme_hw *fme,
+		u64 *val)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+						  FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	*val = readq(&fme_err->seu_emr_h);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +309,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr_low(fme, &prop->data);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr_high(fme, &prop->data);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v6 04/17] raw/ifpga/base: add SEU error support

[Ye Xiaolong]

On 09/19, Andy Pei wrote:
>From: Tianfei zhang <tianfei.zhang@intel.com>
>
>This patch exposes SEU error information to application then application
>could compare this information (128bit) with its own SMH file to know
>if this SEU is a fatal error or not.
>
>Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
>Signed-off-by: Andy Pei <andy.pei@intel.com>
>---
> drivers/raw/ifpga/base/ifpga_defines.h     |  5 +++-
> drivers/raw/ifpga/base/ifpga_fme_error.c   | 43 ++++++++++++++++++++++++++++++
> drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
> 3 files changed, 49 insertions(+), 1 deletion(-)
>
>diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
>index 4216128..b450cb1 100644
>--- a/drivers/raw/ifpga/base/ifpga_defines.h
>+++ b/drivers/raw/ifpga/base/ifpga_defines.h
>@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
> 			u8 support_intr:1;
> 			/* MSI-X vector table entry number */
> 			u16 intr_vector_num:12;
>-			u64 rsvd:51;	/* Reserved */
>+			u64 rsvd:50;	/* Reserved */
>+			u64 seu_support:1;
> 		};
> 	};
> };
>@@ -1171,6 +1172,8 @@ struct feature_fme_err {
> 	struct feature_fme_ras_catfaterror ras_catfaterr;
> 	struct feature_fme_ras_error_inj ras_error_inj;
> 	struct feature_fme_error_capability fme_err_capability;
>+	u64 seu_emr_l;
>+	u64 seu_emr_h;
> };
> 
> /* FME Partial Reconfiguration Control */
>diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
>index a6d3dab..c9bac15 100644
>--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
>+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
>@@ -257,6 +257,45 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
> 	UNUSED(feature);
> }
> 
>+static int fme_err_check_seu(struct feature_fme_err *fme_err)
>+{
>+	struct feature_fme_error_capability error_cap;
>+
>+	error_cap.csr = readq(&fme_err->fme_err_capability);
>+
>+	return error_cap.seu_support ? 1 : 0;
>+}
>+
>+static int fme_err_get_seu_emr_low(struct ifpga_fme_hw *fme,
>+		u64 *val)
>+{
>+	struct feature_fme_err *fme_err
>+		= get_fme_feature_ioaddr_by_index(fme,
>+						  FME_FEATURE_ID_GLOBAL_ERR);
>+
>+	if (!fme_err_check_seu(fme_err))
>+		return -ENODEV;
>+
>+	*val = readq(&fme_err->seu_emr_l);
>+
>+	return 0;
>+}
>+
>+static int fme_err_get_seu_emr_high(struct ifpga_fme_hw *fme,
>+		u64 *val)
>+{
>+	struct feature_fme_err *fme_err
>+		= get_fme_feature_ioaddr_by_index(fme,
>+						  FME_FEATURE_ID_GLOBAL_ERR);
>+
>+	if (!fme_err_check_seu(fme_err))
>+		return -ENODEV;
>+
>+	*val = readq(&fme_err->seu_emr_h);
>+
>+	return 0;
>+}

Above 2 functions can be combined to reduce duplication.

>+
> static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
> 				    struct feature_prop *prop)
> {
>@@ -270,6 +309,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
> 		return fme_err_get_first_error(fme, &prop->data);
> 	case 0x3: /* NEXT_ERROR */
> 		return fme_err_get_next_error(fme, &prop->data);
>+	case 0x5: /* SEU EMR LOW */
>+		return fme_err_get_seu_emr_low(fme, &prop->data);
>+	case 0x6: /* SEU EMR HIGH */
>+		return fme_err_get_seu_emr_high(fme, &prop->data);
> 	}
> 
> 	return -ENOENT;
>diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
>index 4c2c990..bab3386 100644
>--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
>+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
>@@ -74,6 +74,8 @@ struct feature_prop {
> #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
> #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
> #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
>+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
>+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
> #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
> #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
> #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
>-- 
>1.8.3.1
>

Re: [dpdk-dev] [PATCH v6 04/17] raw/ifpga/base: add SEU error support

[Zhang, Tianfei]

> -----Original Message-----
> From: Ye, Xiaolong
> Sent: Wednesday, September 25, 2019 12:37 AM
> To: Pei, Andy <andy.pei@intel.com>
> Cc: dev@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> <tianfei.zhang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Lomartire,
> David <david.lomartire@intel.com>; Yigit, Ferruh <ferruh.yigit@intel.com>
> Subject: Re: [PATCH v6 04/17] raw/ifpga/base: add SEU error support
> 
> On 09/19, Andy Pei wrote:
> >From: Tianfei zhang <tianfei.zhang@intel.com>
> >
> >This patch exposes SEU error information to application then
> >application could compare this information (128bit) with its own SMH
> >file to know if this SEU is a fatal error or not.
> >
> >Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
> >Signed-off-by: Andy Pei <andy.pei@intel.com>
> >---
> > drivers/raw/ifpga/base/ifpga_defines.h     |  5 +++-
> > drivers/raw/ifpga/base/ifpga_fme_error.c   | 43
> ++++++++++++++++++++++++++++++
> > drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
> > 3 files changed, 49 insertions(+), 1 deletion(-)
> >
> >diff --git a/drivers/raw/ifpga/base/ifpga_defines.h
> >b/drivers/raw/ifpga/base/ifpga_defines.h
> >index 4216128..b450cb1 100644
> >--- a/drivers/raw/ifpga/base/ifpga_defines.h
> >+++ b/drivers/raw/ifpga/base/ifpga_defines.h
> >@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
> > 			u8 support_intr:1;
> > 			/* MSI-X vector table entry number */
> > 			u16 intr_vector_num:12;
> >-			u64 rsvd:51;	/* Reserved */
> >+			u64 rsvd:50;	/* Reserved */
> >+			u64 seu_support:1;
> > 		};
> > 	};
> > };
> >@@ -1171,6 +1172,8 @@ struct feature_fme_err {
> > 	struct feature_fme_ras_catfaterror ras_catfaterr;
> > 	struct feature_fme_ras_error_inj ras_error_inj;
> > 	struct feature_fme_error_capability fme_err_capability;
> >+	u64 seu_emr_l;
> >+	u64 seu_emr_h;
> > };
> >
> > /* FME Partial Reconfiguration Control */ diff --git
> >a/drivers/raw/ifpga/base/ifpga_fme_error.c
> >b/drivers/raw/ifpga/base/ifpga_fme_error.c
> >index a6d3dab..c9bac15 100644
> >--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
> >+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
> >@@ -257,6 +257,45 @@ static void fme_global_error_uinit(struct
> ifpga_feature *feature)
> > 	UNUSED(feature);
> > }
> >
> >+static int fme_err_check_seu(struct feature_fme_err *fme_err) {
> >+	struct feature_fme_error_capability error_cap;
> >+
> >+	error_cap.csr = readq(&fme_err->fme_err_capability);
> >+
> >+	return error_cap.seu_support ? 1 : 0; }
> >+
> >+static int fme_err_get_seu_emr_low(struct ifpga_fme_hw *fme,
> >+		u64 *val)
> >+{
> >+	struct feature_fme_err *fme_err
> >+		= get_fme_feature_ioaddr_by_index(fme,
> >+						  FME_FEATURE_ID_GLOBAL_ERR);
> >+
> >+	if (!fme_err_check_seu(fme_err))
> >+		return -ENODEV;
> >+
> >+	*val = readq(&fme_err->seu_emr_l);
> >+
> >+	return 0;
> >+}
> >+
> >+static int fme_err_get_seu_emr_high(struct ifpga_fme_hw *fme,
> >+		u64 *val)
> >+{
> >+	struct feature_fme_err *fme_err
> >+		= get_fme_feature_ioaddr_by_index(fme,
> >+						  FME_FEATURE_ID_GLOBAL_ERR);
> >+
> >+	if (!fme_err_check_seu(fme_err))
> >+		return -ENODEV;
> >+
> >+	*val = readq(&fme_err->seu_emr_h);
> >+
> >+	return 0;
> >+}
> 
> Above 2 functions can be combined to reduce duplication.
> 
Thanks your suggestion, I will fix it in next version.

[dpdk-dev] [PATCH v6 05/17] raw/ifpga/base: add device tree support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 mk/rte.app.mk                             |   2 +-
 3 files changed, 194 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index ba5c39e..2acf9c0 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -319,7 +319,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 06/17] raw/ifpga/base: align the send buffer for SPI

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 07/17] raw/ifpga/base: add sensor support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 08/17] raw/ifpga/base: introducing sensor APIs

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 09/17] raw/ifpga/base: update SEU register definition

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 10/17] raw/ifpga: add SEU error handler

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 245 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index fef89e6..3bcf07b 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -28,6 +28,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -606,6 +608,236 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -653,6 +885,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -678,6 +911,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -697,6 +934,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -721,6 +959,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 11/17] raw/ifpga: add PCIe BDF devices tree scan

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 546 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 557 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 3bcf07b..b36dc88 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_eal_memconfig.h>
@@ -18,7 +20,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -26,6 +28,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -38,6 +41,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -65,6 +74,489 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor is invalid value %x\n",
+					value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev);
+				if (ret == 1)
+					gsd_enable = 1;
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100000 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -373,8 +865,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -843,6 +1336,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -856,7 +1350,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -870,6 +1364,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -988,6 +1490,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1019,13 +1522,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1039,7 +1561,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1066,6 +1589,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 12/17] net/ipn3ke: remove configuration for i40e port bonding

[Andy Pei]

From: Rosen Xu <rosen.xu@intel.com>

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/ipn3ke/Makefile             |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c      | 289 ++++----------------------------
 drivers/net/ipn3ke/ipn3ke_representor.c |   7 +-
 3 files changed, 43 insertions(+), 255 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..2c65e49 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index c226d63..363a5f1 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -241,7 +242,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -259,6 +261,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -323,9 +331,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -376,7 +381,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -403,7 +412,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -415,6 +429,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -422,6 +452,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -467,254 +498,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index 8300cc3..a4ee460 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2906,8 +2907,10 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		i40e_set_switch_dev(rpst->i40e_pf_eth, rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 13/17] raw/ifpga/base: add secure support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 136 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 +++++++++++++-----
 4 files changed, 197 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..e597e47 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,62 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager"))
+			sysmgr_offset = offset;
+			break;
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +575,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 14/17] raw/ifpga/base: configure FEC mode

[Andy Pei]

From: Tianfei Zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 15/17] raw/ifpga/base: clean fme errors

[Andy Pei]

From: Tianfei Zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occurred.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index c9bac15..b7acd17 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index b36dc88..baa3ff7 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1170,6 +1170,25 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1179,6 +1198,9 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 16/17] raw/ifpga/base: add new API get board info

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..e529f54 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disagregate;
+	u8 lightweiht;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..1ebafae 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweiht = id.lightweiht;
+	fme->board_info.disagregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweiht:%s\n"
+			"disagregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweiht),
+			check_support(fme->board_info.disagregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v6 17/17] raw/ifpga: add lightweight fpga image support

[Andy Pei]

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index baa3ff7..f1256d5 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -831,6 +831,8 @@ static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev)
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -857,22 +859,40 @@ static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev)
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweiht) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
-		uuid.b + 8, sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v6 00/17] add PCIe AER disable and IRQ support for ipn3ke

[Ye Xiaolong]

Some general comment, it seems lack of doc/release_note update, and meson
also need to be considered, as it will be the default build system in the future.

Thanks,
Xiaolong

On 09/19, Andy Pei wrote:
>This patch set adds PCIe AER disable and IRQ support for ipn3ke.
>Disable PCIe AER is very useful when FPGA reload. IRQ is used very
>widely in interrupt process.
>
>For ipn3ke is connect to CPU with PCIe switch, driver needs to scan
>all PCIe devices of ipn3ke, it also can get all i40e of card, so
>ipn3ke driver doesn't need to take some configuration of i40e.
>
>v6 update:
>========
>- correct author information.
>- correct typo in commit message and remove Gerrit Change-Id's before
>  submitting upstream
>
>v5 update:
>=========
>- add lightweight fpga image support. in lightweight fpga image mode,
>  ipn3ke representor will not be probed.
>
>v4 updates:
>==========
>- align with new naming standard.
>
>v3 updates:
>===========
>- Add FPGA network side port MTU configuration
>
>v2 updates:
>===========
>- Add AUX feature support
>
>
>Andy Pei (2):
>  net/i40e: i40e support ipn3ke FPGA port bonding
>  raw/ifpga: add lightweight fpga image support
>
>Rosen Xu (3):
>  raw/ifpga: add SEU error handler
>  raw/ifpga: add PCIe BDF devices tree scan
>  net/ipn3ke: remove configuration for i40e port bonding
>
>Tianfei Zhang (2):
>  raw/ifpga/base: configure FEC mode
>  raw/ifpga/base: clean fme errors
>
>Tianfei zhang (10):
>  raw/ifpga/base: add irq support
>  raw/ifpga/base: clear pending bit
>  raw/ifpga/base: add SEU error support
>  raw/ifpga/base: add device tree support
>  raw/ifpga/base: align the send buffer for SPI
>  raw/ifpga/base: add sensor support
>  raw/ifpga/base: introducing sensor APIs
>  raw/ifpga/base: update SEU register definition
>  raw/ifpga/base: add secure support
>  raw/ifpga/base: add new API get board info
>
> drivers/net/i40e/base/i40e_type.h             |   3 +
> drivers/net/i40e/i40e_ethdev.c                |  20 +
> drivers/net/i40e/rte_pmd_i40e.h               |   4 +
> drivers/net/ipn3ke/Makefile                   |   2 +
> drivers/net/ipn3ke/ipn3ke_ethdev.c            | 289 ++-------
> drivers/net/ipn3ke/ipn3ke_representor.c       |   7 +-
> drivers/raw/ifpga/base/ifpga_api.c            |  21 +
> drivers/raw/ifpga/base/ifpga_defines.h        |  75 ++-
> drivers/raw/ifpga/base/ifpga_feature_dev.c    |  60 ++
> drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
> drivers/raw/ifpga/base/ifpga_fme.c            | 138 ++++-
> drivers/raw/ifpga/base/ifpga_fme_error.c      |  89 ++-
> drivers/raw/ifpga/base/ifpga_hw.h             |   2 +-
> drivers/raw/ifpga/base/ifpga_port.c           |  20 +
> drivers/raw/ifpga/base/ifpga_port_error.c     |  21 +
> drivers/raw/ifpga/base/opae_hw_api.c          | 135 ++++
> drivers/raw/ifpga/base/opae_hw_api.h          |  21 +
> drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
> drivers/raw/ifpga/base/opae_intel_max10.c     | 568 ++++++++++++++++-
> drivers/raw/ifpga/base/opae_intel_max10.h     | 146 ++++-
> drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
> drivers/raw/ifpga/base/opae_spi.h             |  23 +-
> drivers/raw/ifpga/base/opae_spi_transaction.c |  40 +-
> drivers/raw/ifpga/ifpga_rawdev.c              | 851 +++++++++++++++++++++++++-
> drivers/raw/ifpga/ifpga_rawdev.h              |  16 +
> mk/rte.app.mk                                 |   2 +-
> 26 files changed, 2187 insertions(+), 378 deletions(-)
>
>-- 
>1.8.3.1
>

Re: [dpdk-dev] [PATCH v6 00/17] add PCIe AER disable and IRQ support for ipn3ke

[Pei, Andy]

Thanks Xiaolong.

Will do that in later version.


-----Original Message-----
From: Ye, Xiaolong 
Sent: Tuesday, September 24, 2019 11:49 PM
To: Pei, Andy <andy.pei@intel.com>
Cc: dev@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Lomartire, David <david.lomartire@intel.com>; Yigit, Ferruh <ferruh.yigit@intel.com>
Subject: Re: [PATCH v6 00/17] add PCIe AER disable and IRQ support for ipn3ke

Some general comment, it seems lack of doc/release_note update, and meson also need to be considered, as it will be the default build system in the future.

Thanks,
Xiaolong

On 09/19, Andy Pei wrote:
>This patch set adds PCIe AER disable and IRQ support for ipn3ke.
>Disable PCIe AER is very useful when FPGA reload. IRQ is used very 
>widely in interrupt process.
>
>For ipn3ke is connect to CPU with PCIe switch, driver needs to scan all 
>PCIe devices of ipn3ke, it also can get all i40e of card, so ipn3ke 
>driver doesn't need to take some configuration of i40e.
>
>v6 update:
>========
>- correct author information.
>- correct typo in commit message and remove Gerrit Change-Id's before
>  submitting upstream
>
>v5 update:
>=========
>- add lightweight fpga image support. in lightweight fpga image mode,
>  ipn3ke representor will not be probed.
>
>v4 updates:
>==========
>- align with new naming standard.
>
>v3 updates:
>===========
>- Add FPGA network side port MTU configuration
>
>v2 updates:
>===========
>- Add AUX feature support
>
>
>Andy Pei (2):
>  net/i40e: i40e support ipn3ke FPGA port bonding
>  raw/ifpga: add lightweight fpga image support
>
>Rosen Xu (3):
>  raw/ifpga: add SEU error handler
>  raw/ifpga: add PCIe BDF devices tree scan
>  net/ipn3ke: remove configuration for i40e port bonding
>
>Tianfei Zhang (2):
>  raw/ifpga/base: configure FEC mode
>  raw/ifpga/base: clean fme errors
>
>Tianfei zhang (10):
>  raw/ifpga/base: add irq support
>  raw/ifpga/base: clear pending bit
>  raw/ifpga/base: add SEU error support
>  raw/ifpga/base: add device tree support
>  raw/ifpga/base: align the send buffer for SPI
>  raw/ifpga/base: add sensor support
>  raw/ifpga/base: introducing sensor APIs
>  raw/ifpga/base: update SEU register definition
>  raw/ifpga/base: add secure support
>  raw/ifpga/base: add new API get board info
>
> drivers/net/i40e/base/i40e_type.h             |   3 +
> drivers/net/i40e/i40e_ethdev.c                |  20 +
> drivers/net/i40e/rte_pmd_i40e.h               |   4 +
> drivers/net/ipn3ke/Makefile                   |   2 +
> drivers/net/ipn3ke/ipn3ke_ethdev.c            | 289 ++-------
> drivers/net/ipn3ke/ipn3ke_representor.c       |   7 +-
> drivers/raw/ifpga/base/ifpga_api.c            |  21 +
> drivers/raw/ifpga/base/ifpga_defines.h        |  75 ++-
> drivers/raw/ifpga/base/ifpga_feature_dev.c    |  60 ++
> drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
> drivers/raw/ifpga/base/ifpga_fme.c            | 138 ++++-
> drivers/raw/ifpga/base/ifpga_fme_error.c      |  89 ++-
> drivers/raw/ifpga/base/ifpga_hw.h             |   2 +-
> drivers/raw/ifpga/base/ifpga_port.c           |  20 +
> drivers/raw/ifpga/base/ifpga_port_error.c     |  21 +
> drivers/raw/ifpga/base/opae_hw_api.c          | 135 ++++
> drivers/raw/ifpga/base/opae_hw_api.h          |  21 +
> drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
> drivers/raw/ifpga/base/opae_intel_max10.c     | 568 ++++++++++++++++-
> drivers/raw/ifpga/base/opae_intel_max10.h     | 146 ++++-
> drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
> drivers/raw/ifpga/base/opae_spi.h             |  23 +-
> drivers/raw/ifpga/base/opae_spi_transaction.c |  40 +-
> drivers/raw/ifpga/ifpga_rawdev.c              | 851 +++++++++++++++++++++++++-
> drivers/raw/ifpga/ifpga_rawdev.h              |  16 +
> mk/rte.app.mk                                 |   2 +-
> 26 files changed, 2187 insertions(+), 378 deletions(-)
>
>--
>1.8.3.1
>

[dpdk-dev] [PATCH v5 02/17] raw/ifpga/base: add irq support

[Andy Pei]

Add irq support for ifpga FME globle error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 60 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 22 +++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 20 ++++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 21 +++++++++++
 4 files changed, 123 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..f0fb242 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,62 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+/*
+ * FIXME: we should get msix vec count during pci enumeration instead of
+ * below hardcode value.
+ */
+#define FPGA_MSIX_VEC_COUNT	20
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	irq_set->count = count;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..068f52c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,31 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set =
+			(struct fpga_fme_err_irq_set *)irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	spinlock_lock(&fme->lock);
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ)) {
+		spinlock_unlock(&fme->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..56b04a6 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,29 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	spinlock_lock(&port->lock);
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ)) {
+		spinlock_unlock(&port->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..8aef7d7 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,30 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	spinlock_lock(&port->lock);
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ)) {
+		spinlock_unlock(&port->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 03/17] raw/ifpga/base: clear pending bit

[Andy Pei]

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 068f52c..a6d3dab 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 04/17] raw/ifpga/base: add SEU error support

[Andy Pei]

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 +++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 43 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index a6d3dab..c9bac15 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,45 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr_low(struct ifpga_fme_hw *fme,
+		u64 *val)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+						  FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
+static int fme_err_get_seu_emr_high(struct ifpga_fme_hw *fme,
+		u64 *val)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+						  FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	*val = readq(&fme_err->seu_emr_h);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +309,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr_low(fme, &prop->data);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr_high(fme, &prop->data);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 05/17] raw/ifpga/base: add device tree support

[Andy Pei]

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 mk/rte.app.mk                             |   2 +-
 3 files changed, 194 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index ba5c39e..2acf9c0 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -319,7 +319,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 06/17] raw/ifpga/base: align the send buffer for SPI

[Andy Pei]

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 07/17] raw/ifpga/base: add sensor support

[Andy Pei]

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 08/17] raw/ifpga/base: introducing sensor APIs

[Andy Pei]

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 09/17] raw/ifpga/base: update SEU register definition

[Andy Pei]

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 10/17] raw/ifpga: add SEU error handler

[Andy Pei]

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 245 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index fef89e6..3bcf07b 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -28,6 +28,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -606,6 +608,236 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -653,6 +885,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -678,6 +911,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -697,6 +934,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -721,6 +959,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 11/17] raw/ifpga: add PCIe BDF devices tree scan

[Andy Pei]

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 546 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 557 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 3bcf07b..b36dc88 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_eal_memconfig.h>
@@ -18,7 +20,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -26,6 +28,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -38,6 +41,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -65,6 +74,489 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor is invalid value %x\n",
+					value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev);
+				if (ret == 1)
+					gsd_enable = 1;
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100000 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -373,8 +865,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -843,6 +1336,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -856,7 +1350,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -870,6 +1364,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -988,6 +1490,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1019,13 +1522,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1039,7 +1561,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1066,6 +1589,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 12/17] net/ipn3ke: remove configuration for i40e port bonding

[Andy Pei]

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/ipn3ke/Makefile             |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c      | 289 ++++----------------------------
 drivers/net/ipn3ke/ipn3ke_representor.c |   7 +-
 3 files changed, 43 insertions(+), 255 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..2c65e49 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index c226d63..363a5f1 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -241,7 +242,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -259,6 +261,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -323,9 +331,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -376,7 +381,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -403,7 +412,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -415,6 +429,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -422,6 +452,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -467,254 +498,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index 8300cc3..a4ee460 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2906,8 +2907,10 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		i40e_set_switch_dev(rpst->i40e_pf_eth, rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 13/17] raw/ifpga/base: add secure support

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add secure max10 device support.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h    |   2 +
 drivers/raw/ifpga/base/ifpga_fme.c        |  26 ++++--
 drivers/raw/ifpga/base/opae_intel_max10.c | 136 +++++++++++++++++++++++++-----
 drivers/raw/ifpga/base/opae_intel_max10.h |  80 +++++++++++++-----
 4 files changed, 197 insertions(+), 47 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 8993cc6..1e84b15 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1698,6 +1698,8 @@ struct ifpga_fme_board_info {
 	u32 patch_version;
 	u32 minor_version;
 	u32 major_version;
+	u32 max10_version;
+	u32 nios_fw_version;
 	u32 nums_of_retimer;
 	u32 ports_per_retimer;
 	u32 nums_of_fvl;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 794ca09..87fa596 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -825,6 +825,7 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	u32 val;
 
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
@@ -850,6 +851,18 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
+	if (max10_sys_read(MAX10_BUILD_VER, &val))
+		return -EINVAL;
+	fme->board_info.max10_version = val & 0xffffff;
+
+	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+		return -EINVAL;
+	fme->board_info.nios_fw_version = val & 0xffffff;
+
+	dev_info(fme, "max10 version 0x%x, nios fw version 0x%x\n",
+		fme->board_info.max10_version,
+		fme->board_info.nios_fw_version);
+
 	return 0;
 }
 
@@ -858,16 +871,11 @@ static int spi_self_checking(void)
 	u32 val;
 	int ret;
 
-	ret = max10_reg_read(0x30043c, &val);
+	ret = max10_sys_read(MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
-	if (val != 0x87654321) {
-		dev_err(NULL, "Read MAX10 test register fail: 0x%x\n", val);
-		return -EIO;
-	}
-
-	dev_info(NULL, "Read MAX10 test register success, SPI self-test done\n");
+	dev_info(NULL, "Read MAX10 test register 0x%x\n", val);
 
 	return 0;
 }
@@ -1283,7 +1291,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1311,7 +1319,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_reg_read(sensor->value_reg, value)) {
+	if (max10_sys_read(sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index ae7a8df..e597e47 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -30,6 +30,22 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+int max10_sys_read(unsigned int offset, unsigned int *val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_read(g_max10->base + offset, val);
+}
+
+int max10_sys_write(unsigned int offset, unsigned int val)
+{
+	if (!g_max10)
+		return -ENODEV;
+
+	return max10_reg_write(g_max10->base + offset, val);
+}
+
 static struct max10_compatible_id max10_id_table[] = {
 	{.compatible = MAX10_PAC,},
 	{.compatible = MAX10_PAC_N3000,},
@@ -66,7 +82,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_NO_I2C2 |
 				MAX10_FLAGS_NO_BMCIMG_FLASH;
 		dev_info(max10, "found %s card\n", max10->id->compatible);
-	}
+	} else
+		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
 static int altera_nor_flash_read(u32 offset,
@@ -100,7 +117,7 @@ static int enable_nor_flash(bool on)
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_reg_read(RSU_REG_OFF, &val);
+	ret = max10_sys_read(RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -111,7 +128,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_reg_write(RSU_REG_OFF, val);
+	return max10_sys_write(RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -123,7 +140,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	ret = max10_sys_read(DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -134,7 +151,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
@@ -315,7 +332,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_reg_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
@@ -355,7 +372,8 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 	return ret;
 }
 
-static int max10_sensor_init(struct intel_max10_device *dev)
+static int
+max10_sensor_init(struct intel_max10_device *dev, int parent)
 {
 	int i, ret = 0, offset = 0;
 	const fdt32_t *num;
@@ -370,7 +388,7 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 		return 0;
 	}
 
-	fdt_for_each_subnode(offset, fdt_root, 0) {
+	fdt_for_each_subnode(offset, fdt_root, parent) {
 		ptr = fdt_get_name(fdt_root, offset, NULL);
 		if (!ptr) {
 			dev_err(dev, "failed to fdt get name\n");
@@ -417,7 +435,16 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 				continue;
 			}
 
-			raw->regs[i].regoff = start;
+			/* This is a hack to compatible with non-secure
+			 * solution. If sensors are included in root node,
+			 * then it's non-secure dtb, which use absolute addr
+			 * of non-secure solution.
+			 */
+			if (parent)
+				raw->regs[i].regoff = start;
+			else
+				raw->regs[i].regoff = start -
+					MAX10_BASE_ADDR;
 			raw->regs[i].size = size;
 		}
 
@@ -469,6 +496,62 @@ static int max10_sensor_init(struct intel_max10_device *dev)
 	return ret;
 }
 
+static int check_max10_version(struct intel_max10_device *dev)
+{
+	unsigned int v;
+
+	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+				&v)) {
+		if (v != 0xffffffff) {
+			dev_info(dev, "secure MAX10 detected\n");
+			dev->base = MAX10_SEC_BASE_ADDR;
+			dev->flags |= MAX10_FLAGS_SECURE;
+		} else {
+			dev_info(dev, "non-secure MAX10 detected\n");
+			dev->base = MAX10_BASE_ADDR;
+		}
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int
+max10_secure_hw_init(struct intel_max10_device *dev)
+{
+	int offset, sysmgr_offset = 0;
+	char *fdt_root;
+
+	fdt_root = dev->fdt_root;
+	if (!fdt_root) {
+		dev_debug(dev, "skip init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		if (!fdt_node_check_compatible(fdt_root, offset,
+					"intel-max10,system-manager"))
+			sysmgr_offset = offset;
+			break;
+	}
+
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, sysmgr_offset);
+
+	return 0;
+}
+
+static int
+max10_non_secure_hw_init(struct intel_max10_device *dev)
+{
+	max10_check_capability(dev);
+
+	max10_sensor_init(dev, 0);
+
+	return 0;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -492,32 +575,47 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
-	/* init the MAX10 device table */
+	/* check the max10 version */
+	ret = check_max10_version(dev);
+	if (ret) {
+		dev_err(dev, "Failed to find max10 hardware!\n");
+		goto free_dev;
+	}
+
+	/* load the MAX10 device table */
 	ret = init_max10_device_table(dev);
 	if (ret) {
-		dev_err(dev, "init max10 device table fail\n");
+		dev_err(dev, "Init max10 device table fail\n");
 		goto free_dev;
 	}
 
-	max10_check_capability(dev);
+	/* init max10 devices, like sensor*/
+	if (dev->flags & MAX10_FLAGS_SECURE)
+		ret = max10_secure_hw_init(dev);
+	else
+		ret = max10_non_secure_hw_init(dev);
+	if (ret) {
+		dev_err(dev, "Failed to init max10 hardware!\n");
+		goto free_dtb;
+	}
 
 	/* read FPGA loading information */
-	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
+	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
-		goto spi_tran_fail;
+		goto release_max10_hw;
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
-
-	max10_sensor_init(dev);
-
 	return dev;
 
-spi_tran_fail:
+release_max10_hw:
+	max10_sensor_uinit();
+free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
-	spi_transaction_remove(dev->spi_tran_dev);
+	if (dev->spi_tran_dev)
+		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
 	opae_free(dev);
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 90bf098..e632941 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -23,6 +23,8 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SPI                 BIT(3)
 #define MAX10_FLGAS_NIOS_SPI            BIT(4)
 #define MAX10_FLAGS_PKVL                BIT(5)
+#define MAX10_FLAGS_SECURE		BIT(6)
+#define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
@@ -30,6 +32,7 @@ struct intel_max10_device {
 	struct spi_transaction_dev *spi_tran_dev;
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
+	unsigned int base; /* max10 base address */
 };
 
 /* retimer speed */
@@ -74,30 +77,69 @@ struct opae_retimer_status {
 #define FLASH_BASE 0x10000000
 #define FLASH_OPTION_BITS 0x10000
 
-#define NIOS2_FW_VERSION_OFF   0x300400
-#define RSU_REG_OFF            0x30042c
-#define FPGA_RP_LOAD		BIT(3)
-#define NIOS2_PRERESET		BIT(4)
-#define NIOS2_HANG		BIT(5)
-#define RSU_ENABLE		BIT(6)
-#define NIOS2_RESET		BIT(7)
-#define NIOS2_I2C2_POLL_STOP	BIT(13)
-#define FPGA_RECONF_REG_OFF	0x300430
-#define COUNTDOWN_START		BIT(18)
-#define MAX10_BUILD_VER_OFF	0x300468
-#define PCB_INFO		GENMASK(31, 24)
-#define MAX10_BUILD_VERION	GENMASK(23, 0)
-#define FPGA_PAGE_INFO_OFF	0x30046c
-#define DT_AVAIL_REG_OFF	0x300490
-#define DT_AVAIL		BIT(0)
-#define DT_BASE_ADDR_REG_OFF	0x300494
-#define PKVL_POLLING_CTRL       0x300480
-#define PKVL_LINK_STATUS        0x300564
+/* System Registers */
+#define MAX10_BASE_ADDR		0x300400
+#define MAX10_SEC_BASE_ADDR	0x300800
+/* Register offset of system registers */
+#define NIOS2_FW_VERSION	0x0
+#define MAX10_MACADDR1		0x10
+#define   MAX10_MAC_BYTE4	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE3	GENMASK(15, 8)
+#define   MAX10_MAC_BYTE2	GENMASK(23, 16)
+#define   MAX10_MAC_BYTE1	GENMASK(31, 24)
+#define MAX10_MACADDR2		0x14
+#define   MAX10_MAC_BYTE6	GENMASK(7, 0)
+#define   MAX10_MAC_BYTE5	GENMASK(15, 8)
+#define   MAX10_MAC_COUNT	GENMASK(23, 16)
+#define RSU_REG			0x2c
+#define   FPGA_RECONF_PAGE	GENMASK(2, 0)
+#define   FPGA_RP_LOAD		BIT(3)
+#define   NIOS2_PRERESET	BIT(4)
+#define   NIOS2_HANG		BIT(5)
+#define   RSU_ENABLE		BIT(6)
+#define   NIOS2_RESET		BIT(7)
+#define   NIOS2_I2C2_POLL_STOP	BIT(13)
+#define   PKVL_EEPROM_LOAD	BIT(31)
+#define FPGA_RECONF_REG		0x30
+#define MAX10_TEST_REG		0x3c
+#define   COUNTDOWN_START	BIT(18)
+#define MAX10_BUILD_VER		0x68
+#define   MAX10_VERSION_MAJOR	GENMASK(23, 16)
+#define   PCB_INFO		GENMASK(31, 24)
+#define FPGA_PAGE_INFO		0x6c
+#define DT_AVAIL_REG		0x90
+#define   DT_AVAIL		BIT(0)
+#define DT_BASE_ADDR_REG	0x94
+#define MAX10_DOORBELL		0x400
+#define   RSU_REQUEST		BIT(0)
+#define   SEC_PROGRESS		GENMASK(7, 4)
+#define   HOST_STATUS		GENMASK(11, 8)
+#define   SEC_STATUS		GENMASK(23, 16)
+
+/* PKVL related registers, in system register region */
+#define PKVL_POLLING_CTRL		0x80
+#define   POLLING_MODE			GENMASK(15, 0)
+#define   PKVL_A_PRELOAD		BIT(16)
+#define   PKVL_A_PRELOAD_TIMEOUT	BIT(17)
+#define   PKVL_A_DATA_TOO_BIG		BIT(18)
+#define   PKVL_A_HDR_CHECKSUM		BIT(20)
+#define   PKVL_B_PRELOAD		BIT(24)
+#define   PKVL_B_PRELOAD_TIMEOUT	BIT(25)
+#define   PKVL_B_DATA_TOO_BIG		BIT(26)
+#define   PKVL_B_HDR_CHECKSUM		BIT(28)
+#define   PKVL_EEPROM_UPG_STATUS	GENMASK(31, 16)
+#define PKVL_LINK_STATUS		0x164
+#define PKVL_A_VERSION			0x254
+#define PKVL_B_VERSION			0x258
+#define   SERDES_VERSION		GENMASK(15, 0)
+#define   SBUS_VERSION			GENMASK(31, 16)
 
 #define DFT_MAX_SIZE		0x7e0000
 
 int max10_reg_read(unsigned int reg, unsigned int *val);
 int max10_reg_write(unsigned int reg, unsigned int val);
+int max10_sys_read(unsigned int offset, unsigned int *val);
+int max10_sys_write(unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 14/17] raw/ifpga/base: configure FEC mode

[Andy Pei]

From: Tianfei Zhang <tianfei.zhang@intel.com>

We can change the PKVL FEC mode when the A10 NIOS FW
initialization. The end-user can use this feature the
change the FEC mode, the default mode is RS FEC mode.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c | 42 +++++++++++++++++++++++++++++---------
 drivers/raw/ifpga/base/opae_spi.h  | 23 +++++++++++++--------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 87fa596..2bc7c10 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -941,9 +941,34 @@ static int nios_spi_wait_init_done(struct altera_spi_device *dev)
 	u32 val = 0;
 	unsigned long timeout = msecs_to_timer_cycles(10000);
 	unsigned long ticks;
+	int major_version;
 
+	if (spi_reg_read(dev, NIOS_VERSION, &val))
+		return -EIO;
+
+	major_version = (val >> NIOS_VERSION_MAJOR_SHIFT) &
+		NIOS_VERSION_MAJOR;
+	dev_debug(dev, "A10 NIOS FW version %d\n", major_version);
+
+	if (major_version >= 3) {
+		/* read NIOS_INIT to check if PKVL INIT done or not */
+		if (spi_reg_read(dev, NIOS_INIT, &val))
+			return -EIO;
+
+		/* check if PKVLs are initialized already */
+		if (val & NIOS_INIT_DONE || val & NIOS_INIT_START)
+			goto nios_init_done;
+
+		/* start to config the default FEC mode */
+		val = NIOS_INIT_START;
+
+		if (spi_reg_write(dev, NIOS_INIT, val))
+			return -EIO;
+	}
+
+nios_init_done:
 	do {
-		if (spi_reg_read(dev, NIOS_SPI_INIT_DONE, &val))
+		if (spi_reg_read(dev, NIOS_INIT, &val))
 			return -EIO;
 		if (val)
 			break;
@@ -961,23 +986,20 @@ static int nios_spi_check_error(struct altera_spi_device *dev)
 {
 	u32 value = 0;
 
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS0, &value))
+	if (spi_reg_read(dev, PKVL_A_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status0 0x%x\n", value);
+	dev_debug(dev, "PKVL A Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
-	value = 0;
-	if (spi_reg_read(dev, NIOS_SPI_INIT_STS1, &value))
+	if (spi_reg_read(dev, PKVL_B_MODE_STS, &value))
 		return -EIO;
 
-	dev_debug(dev, "SPI init status1 0x%x\n", value);
+	dev_debug(dev, "PKVL B Mode Status 0x%x\n", value);
 
-	/* Error code: 0xFFF0 to 0xFFFC */
-	if (value >= 0xFFF0 && value <= 0xFFFC)
+	if (value >= 0x100)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index ab66e1f..6355deb 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -149,12 +149,19 @@ int spi_reg_write(struct altera_spi_device *dev, u32 reg,
 #define NIOS_SPI_STAT 0x18
 #define NIOS_SPI_VALID BIT_ULL(32)
 #define NIOS_SPI_READ_DATA GENMASK_ULL(31, 0)
-#define NIOS_SPI_INIT_DONE 0x1000
-
-#define NIOS_SPI_INIT_DONE 0x1000
-#define NIOS_SPI_INIT_STS0 0x1020
-#define NIOS_SPI_INIT_STS1 0x1024
-#define PKVL_STATUS_RESET  0
-#define PKVL_10G_MODE      1
-#define PKVL_25G_MODE      2
+
+#define NIOS_INIT		0x1000
+#define REQ_FEC_MODE		GENMASK(23, 8)
+#define FEC_MODE_NO		0x0
+#define FEC_MODE_KR		0x5555
+#define FEC_MODE_RS		0xaaaa
+#define NIOS_INIT_START		BIT(1)
+#define NIOS_INIT_DONE		BIT(0)
+#define NIOS_VERSION		0x1004
+#define NIOS_VERSION_MAJOR_SHIFT 28
+#define NIOS_VERSION_MAJOR	GENMASK(31, 28)
+#define NIOS_VERSION_MINOR	GENMASK(27, 24)
+#define NIOS_VERSION_PATCH	GENMASK(23, 20)
+#define PKVL_A_MODE_STS		0x1020
+#define PKVL_B_MODE_STS		0x1024
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 15/17] raw/ifpga/base: clean fme errors

[Andy Pei]

From: Tianfei Zhang <tianfei.zhang@intel.com>

Clean fme errors register when some fme errors occured.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme_error.c | 24 ++----------------------
 drivers/raw/ifpga/ifpga_rawdev.c         | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index c9bac15..b7acd17 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -48,34 +48,14 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	struct feature_fme_err *fme_err
 		= get_fme_feature_ioaddr_by_index(fme,
 						  FME_FEATURE_ID_GLOBAL_ERR);
-	struct feature_fme_error0 fme_error0;
-	struct feature_fme_first_error fme_first_err;
-	struct feature_fme_next_error fme_next_err;
-	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
-
-	fme_error0.csr = readq(&fme_err->fme_err);
-	if (val != fme_error0.csr) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	fme_first_err.csr = readq(&fme_err->fme_first_err);
-	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr, &fme_err->fme_err);
-	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
-	       &fme_err->fme_first_err);
-	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
-	       &fme_err->fme_next_err);
+	writeq(val, &fme_err->fme_err);
 
-exit:
-	writeq(FME_ERROR0_MASK_DEFAULT, &fme_err->fme_err_mask);
 	spinlock_unlock(&fme->lock);
 
-	return ret;
+	return 0;
 }
 
 static int fme_err_get_revision(struct ifpga_fme_hw *fme, u64 *val)
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index b36dc88..baa3ff7 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -1170,6 +1170,25 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	return 0;
 }
 
+static int fme_clean_fme_error(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%lx\n", val);
+
+	ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%lx\n", val);
+
+	return 0;
+}
+
 static int
 fme_err_handle_error0(struct opae_manager *mgr)
 {
@@ -1179,6 +1198,9 @@ static int fme_clear_warning_intr(struct opae_manager *mgr)
 	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
 		return -EINVAL;
 
+	if (fme_clean_fme_error(mgr))
+		return -EINVAL;
+
 	fme_error0.csr = val;
 
 	if (fme_error0.fabric_err)
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 16/17] raw/ifpga/base: add new API get board info

[Andy Pei]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add new API to get the board info.
opae_mgr_get_board_info()

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c     | 11 +++++++
 drivers/raw/ifpga/base/ifpga_defines.h | 55 ++++++++++++++++++++++++++--------
 drivers/raw/ifpga/base/ifpga_fme.c     | 53 +++++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_hw.h      |  2 +-
 drivers/raw/ifpga/base/opae_hw_api.c   | 20 +++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h   |  5 ++++
 6 files changed, 121 insertions(+), 25 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 33d1da3..6dbd715 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -218,10 +218,21 @@ static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
 	return fme_mgr_get_sensor_value(fme, sensor, value);
 }
 
+static int ifpga_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	*info = &fme->board_info;
+
+	return 0;
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
 	.get_sensor_value = ifpga_mgr_get_sensor_value,
+	.get_board_info = ifpga_mgr_get_board_info,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 1e84b15..e529f54 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1667,18 +1667,29 @@ struct bts_header {
 	(((bts_hdr)->guid_h == GBS_GUID_H) &&		\
 	((bts_hdr)->guid_l == GBS_GUID_L))
 
+#define check_support(n) (n == 1 ? "support" : "no")
+
 /* bitstream id definition */
 struct fme_bitstream_id {
 	union {
 		u64 id;
 		struct {
-			u64 hash:32;
-			u64 interface:4;
-			u64 reserved:12;
-			u64 debug:4;
-			u64 patch:4;
-			u64 minor:4;
-			u64 major:4;
+			u8 build_patch:8;
+			u8 build_minor:8;
+			u8 build_major:8;
+			u8 fvl_bypass:1;
+			u8 mac_lightweight:1;
+			u8 disagregate:1;
+			u8 lightweiht:1;
+			u8 seu:1;
+			u8 ptp:1;
+			u8 reserve:2;
+			u8 interface:4;
+			u32 afu_revision:12;
+			u8 patch:4;
+			u8 minor:4;
+			u8 major:4;
+			u8 reserved:4;
 		};
 	};
 };
@@ -1691,13 +1702,31 @@ enum board_interface {
 	VC_2_2_25G = 4,
 };
 
-struct ifpga_fme_board_info {
+enum pac_major {
+	VISTA_CREEK = 0,
+	RUSH_CREEK = 1,
+	DARBY_CREEK = 2,
+};
+
+enum pac_minor {
+	DCP_1_0 = 0,
+	DCP_1_1 = 1,
+	DCP_1_2 = 2,
+};
+
+struct opae_board_info {
+	enum pac_major major;
+	enum pac_minor minor;
 	enum board_interface type;
-	u32 build_hash;
-	u32 debug_version;
-	u32 patch_version;
-	u32 minor_version;
-	u32 major_version;
+
+	/* PAC features */
+	u8 fvl_bypass;
+	u8 mac_lightweight;
+	u8 disagregate;
+	u8 lightweiht;
+	u8 seu;
+	u8 ptp;
+
 	u32 max10_version;
 	u32 nios_fw_version;
 	u32 nums_of_retimer;
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2bc7c10..1ebafae 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -787,8 +787,22 @@ static const char *board_type_to_string(u32 type)
 	return "unknown";
 }
 
+static const char *board_major_to_string(u32 major)
+{
+	switch (major) {
+	case VISTA_CREEK:
+		return "VISTA_CREEK";
+	case RUSH_CREEK:
+		return "RUSH_CREEK";
+	case DARBY_CREEK:
+		return "DARBY_CREEK";
+	}
+
+	return "unknown";
+}
+
 static int board_type_to_info(u32 type,
-		struct ifpga_fme_board_info *info)
+		struct opae_board_info *info)
 {
 	switch (type) {
 	case VC_8_10G:
@@ -830,17 +844,34 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
+	fme->board_info.major = id.major;
+	fme->board_info.minor = id.minor;
 	fme->board_info.type = id.interface;
-	fme->board_info.build_hash = id.hash;
-	fme->board_info.debug_version = id.debug;
-	fme->board_info.major_version = id.major;
-	fme->board_info.minor_version = id.minor;
-
-	dev_info(fme, "board type: %s major_version:%u minor_version:%u build_hash:%u\n",
-			board_type_to_string(fme->board_info.type),
-			fme->board_info.major_version,
-			fme->board_info.minor_version,
-			fme->board_info.build_hash);
+	fme->board_info.fvl_bypass = id.fvl_bypass;
+	fme->board_info.mac_lightweight = id.mac_lightweight;
+	fme->board_info.lightweiht = id.lightweiht;
+	fme->board_info.disagregate = id.disagregate;
+	fme->board_info.seu = id.seu;
+	fme->board_info.ptp = id.ptp;
+
+	dev_info(fme, "found: board: %s type: %s\n",
+			board_major_to_string(fme->board_info.major),
+			board_type_to_string(fme->board_info.type));
+
+	dev_info(fme, "support feature:\n"
+			"fvl_bypass:%s\n"
+			"mac_lightweight:%s\n"
+			"lightweiht:%s\n"
+			"disagregate:%s\n"
+			"seu:%s\n"
+			"ptp1588:%s\n",
+			check_support(fme->board_info.fvl_bypass),
+			check_support(fme->board_info.mac_lightweight),
+			check_support(fme->board_info.lightweiht),
+			check_support(fme->board_info.disagregate),
+			check_support(fme->board_info.seu),
+			check_support(fme->board_info.ptp));
+
 
 	if (board_type_to_info(fme->board_info.type, &fme->board_info))
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/ifpga_hw.h b/drivers/raw/ifpga/base/ifpga_hw.h
index ff91c46..7c3307f 100644
--- a/drivers/raw/ifpga/base/ifpga_hw.h
+++ b/drivers/raw/ifpga/base/ifpga_hw.h
@@ -88,7 +88,7 @@ struct ifpga_fme_hw {
 	void *eth_dev[MAX_ETH_GROUP_DEVICES];
 	struct opae_reg_region
 		eth_group_region[MAX_ETH_GROUP_DEVICES];
-	struct ifpga_fme_board_info board_info;
+	struct opae_board_info board_info;
 	int nums_eth_dev;
 	unsigned int nums_acc_region;
 };
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index d0e66d6..1ccc967 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -690,3 +690,23 @@ struct opae_sensor_info *
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_board_info - get board info
+ * sensor value
+ * @info: opae_board_info for the card
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info)
+{
+	if (!mgr || !info)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_board_info)
+		return mgr->ops->get_board_info(mgr, info);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 0d7be01..b78fbd5 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -13,6 +13,7 @@
 #include "opae_osdep.h"
 #include "opae_intel_max10.h"
 #include "opae_eth_group.h"
+#include "ifpga_defines.h"
 
 #ifndef PCI_MAX_RESOURCE
 #define PCI_MAX_RESOURCE 6
@@ -51,6 +52,8 @@ struct opae_manager_ops {
 	int (*get_sensor_value)(struct opae_manager *mgr,
 			struct opae_sensor_info *sensor,
 			unsigned int *value);
+	int (*get_board_info)(struct opae_manager *mgr,
+			struct opae_board_info **info);
 };
 
 /* networking management ops in FME */
@@ -319,4 +322,6 @@ int opae_manager_eth_group_write_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 data);
 int opae_manager_eth_group_read_reg(struct opae_manager *mgr, u8 group_id,
 		u8 type, u8 index, u16 addr, u32 *data);
+int opae_mgr_get_board_info(struct opae_manager *mgr,
+		struct opae_board_info **info);
 #endif /* _OPAE_HW_API_H_*/
-- 
1.8.3.1

[dpdk-dev] [PATCH v5 17/17] raw/ifpga: add lightweight fpga image support

[Andy Pei]

if fpga image support lightweight feature, set afu uuid to all 0, ipn3ke
representor will not be probed.

Change-Id: Ib3a76fadd0eda8864243da2e73bf5c40d679a3e3
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 44 +++++++++++++++++++++++++++++-----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index baa3ff7..f1256d5 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -831,6 +831,8 @@ static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev)
 	rte_rawdev_obj_t pr_conf)
 {
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_board_info *info;
 	struct rte_afu_pr_conf *afu_pr_conf;
 	int ret;
 	struct uuid uuid;
@@ -857,22 +859,40 @@ static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev)
 		}
 	}
 
-	acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
-	if (!acc)
-		return -ENODEV;
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr) {
+		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
+		return -1;
+	}
 
-	ret = opae_acc_get_uuid(acc, &uuid);
-	if (ret)
-		return ret;
+	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
+		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
+		return -1;
+	}
+
+	if (info->lightweiht) {
+		/* set uuid to all 0, when fpga is lightweight image */
+		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
+		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
+	} else {
+		acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+		if (!acc)
+			return -ENODEV;
 
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
-		uuid.b + 8, sizeof(u64));
+		ret = opae_acc_get_uuid(acc, &uuid);
+		if (ret)
+			return ret;
 
-	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
-		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
+			sizeof(u64));
+		rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
+			sizeof(u64));
 
+		IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
+			__func__,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
+			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
+		}
 	return 0;
 }
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v4 02/12] raw/ifpga/base: add irq support

[Andy Pei]

Add irq support for ifpga FME globle error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_feature_dev.c | 61 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 22 +++++++++++
 drivers/raw/ifpga/base/ifpga_port.c        | 20 ++++++++++
 drivers/raw/ifpga/base/ifpga_port_error.c  | 21 ++++++++++
 4 files changed, 124 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.c b/drivers/raw/ifpga/base/ifpga_feature_dev.c
index 63c8bcc..92d5f93 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,63 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+/*
+ * FIXME: we should get msix vec count during pci enumeration instead of
+ * below hardcode value.
+ */
+#define FPGA_MSIX_VEC_COUNT	20
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	irq_set->count = count;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	opae_memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
+
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 3794564..068f52c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -373,9 +373,31 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set =
+			(struct fpga_fme_err_irq_set *)irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	spinlock_lock(&fme->lock);
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ)) {
+		spinlock_unlock(&fme->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga/base/ifpga_port.c b/drivers/raw/ifpga/base/ifpga_port.c
index 6c41164..56b04a6 100644
--- a/drivers/raw/ifpga/base/ifpga_port.c
+++ b/drivers/raw/ifpga/base/ifpga_port.c
@@ -384,9 +384,29 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	spinlock_lock(&port->lock);
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ)) {
+		spinlock_unlock(&port->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga/base/ifpga_port_error.c b/drivers/raw/ifpga/base/ifpga_port_error.c
index 138284e..8aef7d7 100644
--- a/drivers/raw/ifpga/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga/base/ifpga_port_error.c
@@ -136,9 +136,30 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	spinlock_lock(&port->lock);
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ)) {
+		spinlock_unlock(&port->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v4 03/12] raw/ifpga/base: clear pending bit

[Andy Pei]

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index 068f52c..a6d3dab 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga/base/opae_osdep.h b/drivers/raw/ifpga/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga/base/opae_osdep.h
+++ b/drivers/raw/ifpga/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v4 04/12] raw/ifpga/base: add SEU error support

[Andy Pei]

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h     |  5 +++-
 drivers/raw/ifpga/base/ifpga_fme_error.c   | 43 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga/base/ifpga_fme_error.c b/drivers/raw/ifpga/base/ifpga_fme_error.c
index a6d3dab..c9bac15 100644
--- a/drivers/raw/ifpga/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga/base/ifpga_fme_error.c
@@ -257,6 +257,45 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1 : 0;
+}
+
+static int fme_err_get_seu_emr_low(struct ifpga_fme_hw *fme,
+		u64 *val)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+						  FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
+static int fme_err_get_seu_emr_high(struct ifpga_fme_hw *fme,
+		u64 *val)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+						  FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	*val = readq(&fme_err->seu_emr_h);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +309,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr_low(fme, &prop->data);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr_high(fme, &prop->data);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH v4 05/12] raw/ifpga/base: add device tree support

[Andy Pei]

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 183 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  10 ++
 mk/rte.app.mk                             |   2 +-
 3 files changed, 194 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 9ed10e2..305baba 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index ba5c39e..2acf9c0 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -319,7 +319,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_rawdev_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_rawdev_ifpga -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
-- 
1.8.3.1

[dpdk-dev] [PATCH v4 06/12] raw/ifpga/base: align the send buffer for SPI

[Andy Pei]

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_spi_transaction.c | 40 ++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH v4 07/12] raw/ifpga/base: add sensor support

[Andy Pei]

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/opae_intel_max10.c | 279 ++++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_intel_max10.h |  56 ++++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 305baba..ae7a8df 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH v4 08/12] raw/ifpga/base: introducing sensor APIs

[Andy Pei]

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_api.c         |  10 +++
 drivers/raw/ifpga/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga/base/ifpga_fme.c         |  21 ++++++
 drivers/raw/ifpga/base/opae_hw_api.c       | 115 +++++++++++++++++++++++++++++
 drivers/raw/ifpga/base/opae_hw_api.h       |  16 ++++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga/base/ifpga_api.c b/drivers/raw/ifpga/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga/base/ifpga_api.c
+++ b/drivers/raw/ifpga/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga/base/ifpga_feature_dev.h b/drivers/raw/ifpga/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH v4 09/12] raw/ifpga/base: update SEU register definition

[Andy Pei]

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga/base/ifpga_defines.h b/drivers/raw/ifpga/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga/base/ifpga_defines.h
+++ b/drivers/raw/ifpga/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v4 10/12] raw/ifpga: add SEU error handler

[Andy Pei]

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 245 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index fef89e6..3bcf07b 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -28,6 +28,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -606,6 +608,236 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -653,6 +885,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -678,6 +911,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -697,6 +934,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -721,6 +959,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH v4 11/12] raw/ifpga: add PCIe BDF devices tree scan

[Andy Pei]

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 546 ++++++++++++++++++++++++++++++++++++++-
 drivers/raw/ifpga/ifpga_rawdev.h |  16 ++
 2 files changed, 557 insertions(+), 5 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index 3bcf07b..b36dc88 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_eal_memconfig.h>
@@ -18,7 +20,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -26,6 +28,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -38,6 +41,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR           0x01	/* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE           256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE       4096
+#define RTE_PCI_EXT_CAP_ID(header)       (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)     ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -65,6 +74,489 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	/* scan folder */
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	/* get fpga and fvl */
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+#define AUX_VOLTAGE_WARN 11400
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+		if (ret)
+			goto fail;
+
+		if (value == 0xdeadbeef) {
+			IFPGA_RAWDEV_PMD_ERR("sensor is invalid value %x\n",
+					value);
+			continue;
+		}
+
+		/* monitor temperature sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+			IFPGA_RAWDEV_PMD_INFO("read sensor %s %d %d %d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+
+		/* monitor 12V AUX sensor */
+		if (!strcmp(sensor->name, "12V AUX Voltage")) {
+			if (value < AUX_VOLTAGE_WARN) {
+				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
+						sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		/* get bridge fd */
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		/* save previout ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		/* save previout ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		/* set ECAP_AER+0x08 to 0xFFFFFFFF */
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		/* set ECAP_AER+0x14 to 0xFFFFFFFF */
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		/* read current ECAP_AER+0x08 */
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		/* read current ECAP_AER+0x14 */
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		gsd_enable = 0;
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				ret = set_surprise_link_check_aer(ifpga_rdev);
+				if (ret == 1)
+					gsd_enable = 1;
+			}
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(100000 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR(
+				"Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -373,8 +865,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -843,6 +1336,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -856,7 +1350,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -870,6 +1364,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -988,6 +1490,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1019,13 +1522,32 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	NULL
 };
 
+static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
+	const char *value, void *extra_args)
+{
+	int size;
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	size = strlen(value) + 1;
+	*(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	strlcpy(*(char **)extra_args, value, size);
+
+	return 0;
+}
 static int
 ifpga_cfg_probe(struct rte_vdev_device *dev)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1039,7 +1561,8 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 
 	if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
 		if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
-				       &rte_ifpga_get_string_arg, &name) < 0) {
+				       &ifpga_rawdev_get_string_arg,
+				       &name) < 0) {
 			IFPGA_RAWDEV_PMD_ERR("error to parse %s",
 				     IFPGA_ARG_NAME);
 			goto end;
@@ -1066,6 +1589,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH v4 12/12] net/ipn3ke: remove configuration for i40e port bonding

[Andy Pei]

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/net/ipn3ke/Makefile             |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c      | 289 ++++----------------------------
 drivers/net/ipn3ke/ipn3ke_representor.c |   7 +-
 3 files changed, 43 insertions(+), 255 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..2c65e49 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index c226d63..363a5f1 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -241,7 +242,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -259,6 +261,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -323,9 +331,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -376,7 +381,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -403,7 +412,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -415,6 +429,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -422,6 +452,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -467,254 +498,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index 8300cc3..a4ee460 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2906,8 +2907,10 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		i40e_set_switch_dev(rpst->i40e_pf_eth, rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v4 00/12] Add PCIe AER disable and IRQ support for ipn3ke

[Ye Xiaolong]

Hi, 

There are some compilation and style issues in patchwork about this patchset,
could you help to resolve them?

Thanks,
Xiaolong

On 09/05, Andy Pei wrote:
>This patch set adds PCIe AER disable and IRQ support for ipn3ke.
>Disable PCIe AER is very useful when FPGA reload. IRQ is used very
>widely in interrupt process.
>
>For ipn3ke is connect to CPU with PCIe switch, driver needs to scan
>all PCIe devices of ipn3ke, it also can get all i40e of card, so
>ipn3ke driver doesn't need to take some configuration of i40e.
>
>v4 updates:
>==========
>- align with new naming standard.
>
>v3 updates:
>===========
>- Add FPGA network side port MTU configuration
>
>v2 updates:
>===========
>- Add AUX feature support
>
>Rosen Xu (3):
> net/i40e: i40e support ipn3ke FPGA port bonding
> raw/ifpga: add PCIe BDF devices tree scan
> net/ipn3ke: remove configuration for i40e port bonding
>
>Tianfei Zhang (2):
> raw/ifpga/base: align the send buffer for SPI
> raw/ifpga/base: introducing sensor APIs
>
>Tianfei zhang (7):
> raw/ifpga/base: add irq support
> raw/ifpga/base: clear pending bit
> raw/ifpga/base: add SEU error support
> raw/ifpga/base: add device tree support
> raw/ifpga/base: add sensor support
> raw/ifpga/base: update SEU register definition
> raw/ifpga: add SEU error handler
>
>
> drivers/net/i40e/base/i40e_type.h             |   3 +
> drivers/net/i40e/i40e_ethdev.c                |  32 +-
> drivers/net/i40e/rte_pmd_i40e.h               |   4 +
> drivers/net/ipn3ke/Makefile                   |   2 +
> drivers/net/ipn3ke/ipn3ke_ethdev.c            | 289 ++--------
> drivers/net/ipn3ke/ipn3ke_representor.c       |   7 +-
> drivers/raw/ifpga/base/ifpga_api.c            |  10 +
> drivers/raw/ifpga/base/ifpga_defines.h        |  18 +-
> drivers/raw/ifpga/base/ifpga_feature_dev.c    |  61 ++
> drivers/raw/ifpga/base/ifpga_feature_dev.h    |   3 +
> drivers/raw/ifpga/base/ifpga_fme.c            |  21 +
> drivers/raw/ifpga/base/ifpga_fme_error.c      |  69 ++-
> drivers/raw/ifpga/base/ifpga_port.c           |  20 +
> drivers/raw/ifpga/base/ifpga_port_error.c     |  21 +
> drivers/raw/ifpga/base/opae_hw_api.c          | 115 ++++
> drivers/raw/ifpga/base/opae_hw_api.h          |  16 +
> drivers/raw/ifpga/base/opae_ifpga_hw_api.h    |   2 +
> drivers/raw/ifpga/base/opae_intel_max10.c     | 462 +++++++++++++++
> drivers/raw/ifpga/base/opae_intel_max10.h     |  66 +++
> drivers/raw/ifpga/base/opae_osdep.h           |   7 +-
> drivers/raw/ifpga/base/opae_spi_transaction.c |  40 +-
> drivers/raw/ifpga/ifpga_rawdev.c              | 791 +++++++++++++++++++++++++-
> drivers/raw/ifpga/ifpga_rawdev.h              |  16 +
> mk/rte.app.mk                                 |   2 +-
> 24 files changed, 1799 insertions(+), 278 deletions(-)
>
>-- 
>1.8.3.1
>

[dpdk-dev] [PATCH 02/12] raw/ifpga_rawdev/base: add irq support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add irq support for ifpga FME globle error, port error and uint unit.
We implmented this feature by vfio interrupt mechanism.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c | 61 +++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c   | 22 ++++++++
 drivers/raw/ifpga_rawdev/base/ifpga_port.c        | 20 ++++++++
 drivers/raw/ifpga_rawdev/base/ifpga_port_error.c  | 21 ++++++++
 4 files changed, 124 insertions(+)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
index 63c8bcc..6b942e6 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c
@@ -3,6 +3,7 @@
  */
 
 #include <sys/ioctl.h>
+#include <rte_vfio.h>
 
 #include "ifpga_feature_dev.h"
 
@@ -331,3 +332,63 @@ int port_hw_init(struct ifpga_port_hw *port)
 	port_hw_uinit(port);
 	return ret;
 }
+
+/*
+ * FIXME: we should get msix vec count during pci enumeration instead of
+ * below hardcode value.
+ */
+#define FPGA_MSIX_VEC_COUNT	20
+/* irq set buffer length for interrupt */
+#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
+				sizeof(int) * FPGA_MSIX_VEC_COUNT)
+
+/* only support msix for now*/
+static int vfio_msix_enable_block(s32 vfio_dev_fd, unsigned int vec_start,
+				  unsigned int count, s32 *fds)
+{
+	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
+	struct vfio_irq_set *irq_set;
+	int len, ret;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	irq_set->count = count;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+				VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = vec_start;
+
+	fd_ptr = (int *)&irq_set->data;
+	memcpy(fd_ptr, fds, sizeof(int) * count);
+
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		printf("Error enabling MSI-X interrupts\n");
+
+	return ret;
+}
+
+int fpga_msix_set_block(struct ifpga_feature *feature, unsigned int start,
+			unsigned int count, s32 *fds)
+{
+	struct feature_irq_ctx *ctx = feature->ctx;
+	unsigned int i;
+	int ret;
+
+	if (start >= feature->ctx_num || start + count > feature->ctx_num)
+		return -EINVAL;
+
+	/* assume that each feature has continuous vector space in msix*/
+	ret = vfio_msix_enable_block(feature->vfio_dev_fd,
+				     ctx[start].idx, count, fds);
+	if (!ret) {
+		for (i = 0; i < count; i++)
+			ctx[i].eventfd = fds[i];
+	}
+
+	return ret;
+}
+
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
index 3794564..068f52c 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
@@ -373,9 +373,31 @@ static int fme_global_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int fme_global_err_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_fme_err_irq_set *err_irq_set =
+			(struct fpga_fme_err_irq_set *)irq_set;
+	struct ifpga_fme_hw *fme;
+	int ret;
+
+	fme = (struct ifpga_fme_hw *)feature->parent;
+
+	spinlock_lock(&fme->lock);
+	if (!(fme->capability & FPGA_FME_CAP_ERR_IRQ)) {
+		spinlock_unlock(&fme->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&fme->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops fme_global_err_ops = {
 	.init = fme_global_error_init,
 	.uinit = fme_global_error_uinit,
 	.get_prop = fme_global_error_get_prop,
 	.set_prop = fme_global_error_set_prop,
+	.set_irq = fme_global_err_set_irq,
 };
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_port.c b/drivers/raw/ifpga_rawdev/base/ifpga_port.c
index 6c41164..56b04a6 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_port.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_port.c
@@ -384,9 +384,29 @@ static void port_uint_uinit(struct ifpga_feature *feature)
 	dev_info(NULL, "PORT UINT UInit.\n");
 }
 
+static int port_uint_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_uafu_irq_set *uafu_irq_set = irq_set;
+	struct ifpga_port_hw *port = feature->parent;
+	int ret;
+
+	spinlock_lock(&port->lock);
+	if (!(port->capability & FPGA_PORT_CAP_UAFU_IRQ)) {
+		spinlock_unlock(&port->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, uafu_irq_set->start,
+				  uafu_irq_set->count, uafu_irq_set->evtfds);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_uint_ops = {
 	.init = port_uint_init,
 	.uinit = port_uint_uinit,
+	.set_irq = port_uint_set_irq,
 };
 
 static int port_afu_init(struct ifpga_feature *feature)
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c b/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c
index 138284e..8aef7d7 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c
@@ -136,9 +136,30 @@ static int port_error_set_prop(struct ifpga_feature *feature,
 	return -ENOENT;
 }
 
+static int port_error_set_irq(struct ifpga_feature *feature, void *irq_set)
+{
+	struct fpga_port_err_irq_set *err_irq_set = irq_set;
+	struct ifpga_port_hw *port;
+	int ret;
+
+	port = feature->parent;
+
+	spinlock_lock(&port->lock);
+	if (!(port->capability & FPGA_PORT_CAP_ERR_IRQ)) {
+		spinlock_unlock(&port->lock);
+		return -ENODEV;
+	}
+
+	ret = fpga_msix_set_block(feature, 0, 1, &err_irq_set->evtfd);
+	spinlock_unlock(&port->lock);
+
+	return ret;
+}
+
 struct ifpga_feature_ops ifpga_rawdev_port_error_ops = {
 	.init = port_error_init,
 	.uinit = port_error_uinit,
 	.get_prop = port_error_get_prop,
 	.set_prop = port_error_set_prop,
+	.set_irq = port_error_set_irq,
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH 03/12] raw/ifpga_rawdev/base: clear pending bit

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Every defined bit in FME_ERROR0 is RW1C. Other reserved bits are always
0 when readout and it will plan to be RW1C if needed in future.
So it is safe just write the read back value to clear all the errors.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_defines.h   | 9 ++++-----
 drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c | 4 ++--
 drivers/raw/ifpga_rawdev/base/opae_osdep.h      | 7 +++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
index b7151ca..4216128 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
@@ -957,25 +957,24 @@ struct feature_fme_dperf {
 };
 
 struct feature_fme_error0 {
-#define FME_ERROR0_MASK        0xFFUL
 #define FME_ERROR0_MASK_DEFAULT 0x40UL  /* pcode workaround */
 	union {
 		u64 csr;
 		struct {
 			u8  fabric_err:1;	/* Fabric error */
 			u8  fabfifo_overflow:1;	/* Fabric fifo overflow */
-			u8  kticdc_parity_err:2;/* KTI CDC Parity Error */
-			u8  iommu_parity_err:1;	/* IOMMU Parity error */
+			u8  reserved2:3;
 			/* AFU PF/VF access mismatch detected */
 			u8  afu_acc_mode_err:1;
-			u8  mbp_err:1;		/* Indicates an MBP event */
+			u8  reserved6:1;
 			/* PCIE0 CDC Parity Error */
 			u8  pcie0cdc_parity_err:5;
 			/* PCIE1 CDC Parity Error */
 			u8  pcie1cdc_parity_err:5;
 			/* CVL CDC Parity Error */
 			u8  cvlcdc_parity_err:3;
-			u64 rsvd:44;		/* Reserved */
+			u8  fpgaseuerr:1;
+			u64 rsvd:43;		/* Reserved */
 		};
 	};
 };
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
index 068f52c..a6d3dab 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
@@ -54,7 +54,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	int ret = 0;
 
 	spinlock_lock(&fme->lock);
-	writeq(FME_ERROR0_MASK, &fme_err->fme_err_mask);
+	writeq(GENMASK_ULL(63, 0), &fme_err->fme_err_mask);
 
 	fme_error0.csr = readq(&fme_err->fme_err);
 	if (val != fme_error0.csr) {
@@ -65,7 +65,7 @@ static int fme_err_set_clear(struct ifpga_fme_hw *fme, u64 val)
 	fme_first_err.csr = readq(&fme_err->fme_first_err);
 	fme_next_err.csr = readq(&fme_err->fme_next_err);
 
-	writeq(fme_error0.csr & FME_ERROR0_MASK, &fme_err->fme_err);
+	writeq(fme_error0.csr, &fme_err->fme_err);
 	writeq(fme_first_err.csr & FME_FIRST_ERROR_MASK,
 	       &fme_err->fme_first_err);
 	writeq(fme_next_err.csr & FME_NEXT_ERROR_MASK,
diff --git a/drivers/raw/ifpga_rawdev/base/opae_osdep.h b/drivers/raw/ifpga_rawdev/base/opae_osdep.h
index 1596adc..416cef0 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_osdep.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_osdep.h
@@ -32,10 +32,12 @@ struct uuid {
 #ifndef BITS_PER_LONG
 #define BITS_PER_LONG	(__SIZEOF_LONG__ * 8)
 #endif
+#ifndef BITS_PER_LONG_LONG
+#define BITS_PER_LONG_LONG  (__SIZEOF_LONG_LONG__ * 8)
+#endif
 #ifndef BIT
 #define BIT(a) (1UL << (a))
 #endif /* BIT */
-#define U64_C(x) x ## ULL
 #ifndef BIT_ULL
 #define BIT_ULL(a) (1ULL << (a))
 #endif /* BIT_ULL */
@@ -43,7 +45,8 @@ struct uuid {
 #define GENMASK(h, l)	(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif /* GENMASK */
 #ifndef GENMASK_ULL
-#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l))
+#define GENMASK_ULL(h, l) \
+	(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
 #endif /* GENMASK_ULL */
 #endif /* LINUX_MACROS */
 
-- 
1.8.3.1

[dpdk-dev] [PATCH 04/12] raw/ifpga_rawdev/base: add SEU error support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

This patch exposes SEU error information to application then application
could compare this information (128bit) with its own SMH file to know
if this SEU is a fatal error or not.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_defines.h     |  5 ++-
 drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c   | 43 +++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h |  2 ++
 3 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
index 4216128..b450cb1 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
@@ -1149,7 +1149,8 @@ struct feature_fme_error_capability {
 			u8 support_intr:1;
 			/* MSI-X vector table entry number */
 			u16 intr_vector_num:12;
-			u64 rsvd:51;	/* Reserved */
+			u64 rsvd:50;	/* Reserved */
+			u64 seu_support:1;
 		};
 	};
 };
@@ -1171,6 +1172,8 @@ struct feature_fme_err {
 	struct feature_fme_ras_catfaterror ras_catfaterr;
 	struct feature_fme_ras_error_inj ras_error_inj;
 	struct feature_fme_error_capability fme_err_capability;
+	u64 seu_emr_l;
+	u64 seu_emr_h;
 };
 
 /* FME Partial Reconfiguration Control */
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
index a6d3dab..b496667 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c
@@ -257,6 +257,45 @@ static void fme_global_error_uinit(struct ifpga_feature *feature)
 	UNUSED(feature);
 }
 
+static int fme_err_check_seu(struct feature_fme_err *fme_err)
+{
+	struct feature_fme_error_capability error_cap;
+
+	error_cap.csr = readq(&fme_err->fme_err_capability);
+
+	return error_cap.seu_support ? 1:0;
+}
+
+static int fme_err_get_seu_emr_low(struct ifpga_fme_hw *fme,
+		u64 *val)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+						  FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	*val = readq(&fme_err->seu_emr_l);
+
+	return 0;
+}
+
+static int fme_err_get_seu_emr_high(struct ifpga_fme_hw *fme,
+		u64 *val)
+{
+	struct feature_fme_err *fme_err
+		= get_fme_feature_ioaddr_by_index(fme,
+						  FME_FEATURE_ID_GLOBAL_ERR);
+
+	if (!fme_err_check_seu(fme_err))
+		return -ENODEV;
+
+	*val = readq(&fme_err->seu_emr_h);
+
+	return 0;
+}
+
 static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 				    struct feature_prop *prop)
 {
@@ -270,6 +309,10 @@ static int fme_err_fme_err_get_prop(struct ifpga_feature *feature,
 		return fme_err_get_first_error(fme, &prop->data);
 	case 0x3: /* NEXT_ERROR */
 		return fme_err_get_next_error(fme, &prop->data);
+	case 0x5: /* SEU EMR LOW */
+		return fme_err_get_seu_emr_low(fme, &prop->data);
+	case 0x6: /* SEU EMR HIGH */
+		return fme_err_get_seu_emr_high(fme, &prop->data);
 	}
 
 	return -ENOENT;
diff --git a/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h b/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h
index 4c2c990..bab3386 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h
@@ -74,6 +74,8 @@ struct feature_prop {
 #define FME_ERR_PROP_FIRST_ERROR	ERR_PROP_FME_ERR(0x2)
 #define FME_ERR_PROP_NEXT_ERROR		ERR_PROP_FME_ERR(0x3)
 #define FME_ERR_PROP_CLEAR		ERR_PROP_FME_ERR(0x4)	/* WO */
+#define FME_ERR_PROP_SEU_EMR_LOW        ERR_PROP_FME_ERR(0x5)
+#define FME_ERR_PROP_SEU_EMR_HIGH       ERR_PROP_FME_ERR(0x6)
 #define FME_ERR_PROP_REVISION		ERR_PROP_ROOT(0x5)
 #define FME_ERR_PROP_PCIE0_ERRORS	ERR_PROP_ROOT(0x6)	/* RW */
 #define FME_ERR_PROP_PCIE1_ERRORS	ERR_PROP_ROOT(0x7)	/* RW */
-- 
1.8.3.1

[dpdk-dev] [PATCH 05/12] raw/ifpga_rawdev/base: add device tree support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, this is a BMC chip which using MAX10 FPGA
to manage the board configuration, like sensors, flash controller,
QSFP, powers. And this is a SPI bus connected between A10 FPGA and
MAX10, we can access the MAX10 registers over this SPI bus.

In BMC, there are about 19 sensors in MAX10 chip, including the FPGA
core temperature, Board temperature, board current, voltage and so on.

We use DTB (Device tree table) to describe it. This DTB file is store
in nor flash partition, which will flashed in Factory when the boards
delivery to customers. And the same time, the customers can easy to
customizate the BMC configuration like change the sensors.

Add device tree support by using libfdt library in Linux distribution.
The end-user should pre-install the libfdt and libfdt-devel package
before use DPDK on PAC N3000 Card.

For Centos 7.x: sudo yum install libfdt libfdt-devel
For Ubuntu 18.04: sudo apt install libfdt-dev libfdt1

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.c | 183 +++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.h |  10 ++
 mk/rte.app.mk                                    |   2 +-
 3 files changed, 194 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
index 9ed10e2..8617173 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
@@ -3,6 +3,7 @@
  */
 
 #include "opae_intel_max10.h"
+#include <libfdt.h>
 
 static struct intel_max10_device *g_max10;
 
@@ -26,6 +27,174 @@ int max10_reg_write(unsigned int reg, unsigned int val)
 			reg, 4, (unsigned char *)&tmp);
 }
 
+static struct max10_compatible_id max10_id_table[] = {
+	{.compatible = MAX10_PAC,},
+	{.compatible = MAX10_PAC_N3000,},
+	{.compatible = MAX10_PAC_END,}
+};
+
+static struct max10_compatible_id *max10_match_compatible(const char *fdt_root)
+{
+	struct max10_compatible_id *id = max10_id_table;
+
+	for (; strcmp(id->compatible, MAX10_PAC_END); id++) {
+		if (fdt_node_check_compatible(fdt_root, 0, id->compatible))
+			continue;
+
+		return id;
+	}
+
+	return NULL;
+}
+
+static inline bool
+is_max10_pac_n3000(struct intel_max10_device *max10)
+{
+	return max10->id && !strcmp(max10->id->compatible,
+			MAX10_PAC_N3000);
+}
+
+static void max10_check_capability(struct intel_max10_device *max10)
+{
+	if (!max10->fdt_root)
+		return;
+
+	if (is_max10_pac_n3000(max10)) {
+		max10->flags |= MAX10_FLAGS_NO_I2C2 |
+				MAX10_FLAGS_NO_BMCIMG_FLASH;
+		dev_info(max10, "found %s card\n", max10->id->compatible);
+	}
+}
+
+static int altera_nor_flash_read(u32 offset,
+		void *buffer, u32 len)
+{
+	int word_len;
+	int i;
+	unsigned int *buf = (unsigned int *)buffer;
+	unsigned int value;
+	int ret;
+
+	if (!buffer || len <= 0)
+		return -ENODEV;
+
+	word_len = len/4;
+
+	for (i = 0; i < word_len; i++) {
+		ret = max10_reg_read(FLASH_BASE + offset + i*4,
+				&value);
+		if (ret)
+			return -EBUSY;
+
+		*buf++ = value;
+	}
+
+	return 0;
+}
+
+static int enable_nor_flash(bool on)
+{
+	unsigned int val = 0;
+	int ret;
+
+	ret = max10_reg_read(RSU_REG_OFF, &val);
+	if (ret) {
+		dev_err(NULL "enabling flash error\n");
+		return ret;
+	}
+
+	if (on)
+		val |= RSU_ENABLE;
+	else
+		val &= ~RSU_ENABLE;
+
+	return max10_reg_write(RSU_REG_OFF, val);
+}
+
+static int init_max10_device_table(struct intel_max10_device *max10)
+{
+	struct max10_compatible_id *id;
+	struct fdt_header hdr;
+	char *fdt_root = NULL;
+
+	u32 dt_size, dt_addr, val;
+	int ret;
+
+	ret = max10_reg_read(DT_AVAIL_REG_OFF, &val);
+	if (ret) {
+		dev_err(max10 "cannot read DT_AVAIL_REG\n");
+		return ret;
+	}
+
+	if (!(val & DT_AVAIL)) {
+		dev_err(max10 "DT not available\n");
+		return -EINVAL;
+	}
+
+	ret = max10_reg_read(DT_BASE_ADDR_REG_OFF, &dt_addr);
+	if (ret) {
+		dev_info(max10 "cannot get base addr of device table\n");
+		return ret;
+	}
+
+	ret = enable_nor_flash(true);
+	if (ret) {
+		dev_err(max10 "fail to enable flash\n");
+		return ret;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	if (ret) {
+		dev_err(max10 "read fdt header fail\n");
+		goto done;
+	}
+
+	ret = fdt_check_header(&hdr);
+	if (ret) {
+		dev_err(max10 "check fdt header fail\n");
+		goto done;
+	}
+
+	dt_size = fdt_totalsize(&hdr);
+	if (dt_size > DFT_MAX_SIZE) {
+		dev_err(max10 "invalid device table size\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	fdt_root = opae_malloc(dt_size);
+	if (!fdt_root) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	if (ret) {
+		dev_err(max10 "cannot read device table\n");
+		goto done;
+	}
+
+	id = max10_match_compatible(fdt_root);
+	if (!id) {
+		dev_err(max10 "max10 compatible not found\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	max10->flags |= MAX10_FLAGS_DEVICE_TABLE;
+
+	max10->id = id;
+	max10->fdt_root = fdt_root;
+
+done:
+	ret = enable_nor_flash(false);
+
+	if (ret && fdt_root)
+		opae_free(fdt_root);
+
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -49,6 +218,15 @@ struct intel_max10_device *
 	/* set the max10 device firstly */
 	g_max10 = dev;
 
+	/* init the MAX10 device table */
+	ret = init_max10_device_table(dev);
+	if (ret) {
+		dev_err(dev, "init max10 device table fail\n");
+		goto free_dev;
+	}
+
+	max10_check_capability(dev);
+
 	/* read FPGA loading information */
 	ret = max10_reg_read(FPGA_PAGE_INFO_OFF, &val);
 	if (ret) {
@@ -60,6 +238,8 @@ struct intel_max10_device *
 	return dev;
 
 spi_tran_fail:
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
 	spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
 	g_max10 = NULL;
@@ -76,6 +256,9 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
+	if (dev->fdt_root)
+		opae_free(dev->fdt_root);
+
 	g_max10 = NULL;
 	opae_free(dev);
 
diff --git a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
index 08b387e..a52b63e 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
@@ -8,6 +8,14 @@
 #include "opae_osdep.h"
 #include "opae_spi.h"
 
+struct max10_compatible_id {
+	char compatible[128];
+};
+
+#define MAX10_PAC	"intel,max10"
+#define MAX10_PAC_N3000	"intel,max10-pac-n3000"
+#define MAX10_PAC_END    "intel,end"
+
 /* max10 capability flags */
 #define MAX10_FLAGS_NO_I2C2		BIT(0)
 #define MAX10_FLAGS_NO_BMCIMG_FLASH	BIT(1)
@@ -20,6 +28,8 @@ struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
 	struct spi_transaction_dev *spi_tran_dev;
+	struct max10_compatible_id *id; /*max10 compatible*/
+	char *fdt_root;
 };
 
 /* retimer speed */
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index a277c80..c880506 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -319,7 +319,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_pmd_dpaa2_qdma
 endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS)      += -lrte_bus_ifpga
 ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
-_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_pmd_ifpga_rawdev
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_pmd_ifpga_rawdev -lfdt
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_pmd_ioat_rawdev
-- 
1.8.3.1

[dpdk-dev] [PATCH 06/12] raw/ifpga_rawdev/base: align the send buffer for SPI

[Rosen Xu]

From: Tianfei Zhang <tianfei.zhang@intel.com>

The length of send buffer of SPI bus should be 4bytes align.

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
---
 .../raw/ifpga_rawdev/base/opae_spi_transaction.c   | 40 +++++++++++++++++++---
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/raw/ifpga_rawdev/base/opae_spi_transaction.c b/drivers/raw/ifpga_rawdev/base/opae_spi_transaction.c
index 17ec3c1..06ca625 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga_rawdev/base/opae_spi_transaction.c
@@ -109,6 +109,34 @@ static int resp_find_sop_eop(unsigned char *resp, unsigned int len,
 	return ret;
 }
 
+static void phy_tx_pad(unsigned char *phy_buf, unsigned int phy_buf_len,
+		unsigned int *aligned_len)
+{
+	unsigned char *p = &phy_buf[phy_buf_len - 1], *dst_p;
+
+	*aligned_len = IFPGA_ALIGN(phy_buf_len, 4);
+
+	if (*aligned_len == phy_buf_len)
+		return;
+
+	dst_p = &phy_buf[*aligned_len - 1];
+
+	/* move EOP and bytes after EOP to the end of aligned size */
+	while (p > phy_buf) {
+		*dst_p = *p;
+
+		if (*p == SPI_PACKET_EOP)
+			break;
+
+		p--;
+		dst_p--;
+	}
+
+	/* fill the hole with PHY_IDLE */
+	while (p < dst_p)
+		*p++ = SPI_BYTE_IDLE;
+}
+
 static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		unsigned int send_len, unsigned char *send_data,
 		unsigned int resp_len, unsigned char *resp_data,
@@ -149,15 +177,19 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 		}
 	}
 
-	print_buffer("before spi:", send_packet, p-send_packet);
+	tx_len = p - send_packet;
+
+	print_buffer("before spi:", send_packet, tx_len);
 
-	reorder_phy_data(32, send_packet, p - send_packet);
+	phy_tx_pad(send_packet, tx_len, &tx_len);
+	print_buffer("after pad:", send_packet, tx_len);
 
-	print_buffer("after order to spi:", send_packet, p-send_packet);
+	reorder_phy_data(32, send_packet, tx_len);
+
+	print_buffer("after order to spi:", send_packet, tx_len);
 
 	/* call spi */
 	tx_buffer = send_packet;
-	tx_len = p - send_packet;
 	rx_buffer = resp_packet;
 	rx_len = resp_max_len;
 	spi_flags = SPI_NOT_FOUND;
-- 
1.8.3.1

[dpdk-dev] [PATCH 07/12] raw/ifpga_rawdev/base: add sensor support

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

The sensor devices are connected in MAX10 FPGA. we used the
device tree to describe those sensor devices. Parse the device
tree to get the sensor devices and add them into a list.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.c | 279 +++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_intel_max10.h |  56 +++++
 2 files changed, 335 insertions(+)

diff --git a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
index 8617173..474941e 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.c
@@ -7,6 +7,9 @@
 
 static struct intel_max10_device *g_max10;
 
+struct opae_sensor_list opae_sensor_list =
+	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
+
 int max10_reg_read(unsigned int reg, unsigned int *val)
 {
 	if (!g_max10)
@@ -195,6 +198,277 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	return ret;
 }
 
+static u64 fdt_get_number(const fdt32_t *cell, int size)
+{
+	u64 r = 0;
+
+	while (size--)
+		r = (r << 32) | fdt32_to_cpu(*cell++);
+
+	return r;
+}
+
+static int fdt_get_reg(const void *fdt, int node, unsigned int idx,
+		u64 *start, u64 *size)
+{
+	const fdt32_t *prop, *end;
+	int na = 0, ns = 0, len = 0, parent;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+
+	prop = fdt_getprop(fdt, parent, "#address-cells", NULL);
+	na = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, parent, "#size-cells", NULL);
+	ns = prop ? fdt32_to_cpu(*prop) : 2;
+
+	prop = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop)
+		return -FDT_ERR_NOTFOUND;
+
+	end = prop + len/sizeof(*prop);
+	prop = prop + (na + ns) * idx;
+
+	if (prop + na + ns > end)
+		return -FDT_ERR_NOTFOUND;
+
+	*start = fdt_get_number(prop, na);
+	*size = fdt_get_number(prop + na, ns);
+
+	return 0;
+}
+
+static int __fdt_stringlist_search(const void *fdt, int offset,
+		const char *prop, const char *string)
+{
+	int length, len, index = 0;
+	const char *list, *end;
+
+	list = fdt_getprop(fdt, offset, prop, &length);
+	if (!list)
+		return length;
+
+	len = strlen(string) + 1;
+	end = list + length;
+
+	while (list < end) {
+		length = strnlen(list, end - list) + 1;
+
+		if (list + length > end)
+			return -FDT_ERR_BADVALUE;
+
+		if (length == len && memcmp(list, string, length) == 0)
+			return index;
+
+		list += length;
+		index++;
+	}
+
+	return -FDT_ERR_NOTFOUND;
+}
+
+static int fdt_get_named_reg(const void *fdt, int node, const char *name,
+		u64 *start, u64 *size)
+{
+	int idx;
+
+	idx = __fdt_stringlist_search(fdt, node, "reg-names", name);
+	if (idx < 0)
+		return idx;
+
+	return fdt_get_reg(fdt, node, idx, start, size);
+}
+
+static void max10_sensor_uinit(void)
+{
+	struct opae_sensor_info *info;
+
+	TAILQ_FOREACH(info, &opae_sensor_list, node) {
+		TAILQ_REMOVE(&opae_sensor_list, info, node);
+		opae_free(info);
+	}
+}
+
+static bool sensor_reg_valid(struct sensor_reg *reg)
+{
+	return !!reg->size;
+}
+
+static int max10_add_sensor(struct raw_sensor_info *info,
+		struct opae_sensor_info *sensor)
+{
+	int i;
+	int ret = 0;
+	unsigned int val;
+
+	if (!info || !sensor)
+		return -ENODEV;
+
+	sensor->id = info->id;
+	sensor->name = info->name;
+	sensor->type = info->type;
+	sensor->multiplier = info->multiplier;
+
+	for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+		if (!sensor_reg_valid(&info->regs[i]))
+			continue;
+
+		ret = max10_reg_read(info->regs[i].regoff, &val);
+		if (ret)
+			break;
+
+		if (val == 0xdeadbeef)
+			continue;
+
+		val *= info->multiplier;
+
+		switch (i) {
+		case SENSOR_REG_VALUE:
+			sensor->value_reg = info->regs[i].regoff;
+			sensor->flags |= OPAE_SENSOR_VALID;
+			break;
+		case SENSOR_REG_HIGH_WARN:
+			sensor->high_warn = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_WARN_VALID;
+			break;
+		case SENSOR_REG_HIGH_FATAL:
+			sensor->high_fatal = val;
+			sensor->flags |= OPAE_SENSOR_HIGH_FATAL_VALID;
+			break;
+		case SENSOR_REG_LOW_WARN:
+			sensor->low_warn = val;
+			sensor->flags |= OPAE_SENSOR_LOW_WARN_VALID;
+			break;
+		case SENSOR_REG_LOW_FATAL:
+			sensor->low_fatal = val;
+			sensor->flags |= OPAE_SENSOR_LOW_FATAL_VALID;
+			break;
+		case SENSOR_REG_HYSTERESIS:
+			sensor->hysteresis = val;
+			sensor->flags |= OPAE_SENSOR_HYSTERESIS_VALID;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int max10_sensor_init(struct intel_max10_device *dev)
+{
+	int i, ret = 0, offset = 0;
+	const fdt32_t *num;
+	const char *ptr;
+	u64 start, size;
+	struct raw_sensor_info *raw;
+	struct opae_sensor_info *sensor;
+	char *fdt_root = dev->fdt_root;
+
+	if (!fdt_root) {
+		dev_debug(dev, "skip sensor init as not find Device Tree\n");
+		return 0;
+	}
+
+	fdt_for_each_subnode(offset, fdt_root, 0) {
+		ptr = fdt_get_name(fdt_root, offset, NULL);
+		if (!ptr) {
+			dev_err(dev, "failed to fdt get name\n");
+			continue;
+		}
+
+		if (!strstr(ptr, "sensor")) {
+			dev_debug(dev, "%s is not a sensor node\n", ptr);
+			continue;
+		}
+
+		dev_debug(dev, "found sensor node %s\n", ptr);
+
+		raw = (struct raw_sensor_info *)opae_zmalloc(sizeof(*raw));
+		if (!raw) {
+			ret = -ENOMEM;
+			goto free_sensor;
+		}
+
+		raw->name = fdt_getprop(fdt_root, offset, "sensor_name", NULL);
+		if (!raw->name) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->type = fdt_getprop(fdt_root, offset, "type", NULL);
+		if (!raw->type) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++) {
+			ret = fdt_get_named_reg(fdt_root, offset,
+					sensor_reg_name[i], &start,
+					&size);
+			if (ret) {
+				dev_debug(dev, "no found %d: sensor node %s, %s\n",
+						ret, ptr, sensor_reg_name[i]);
+				if (i == SENSOR_REG_VALUE) {
+					ret = -EINVAL;
+					goto free_sensor;
+				}
+
+				continue;
+			}
+
+			raw->regs[i].regoff = start;
+			raw->regs[i].size = size;
+		}
+
+		num = fdt_getprop(fdt_root, offset, "id", NULL);
+		if (!num) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		raw->id = fdt32_to_cpu(*num);
+		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
+		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
+
+		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+				raw->name, raw->type,
+				raw->id, raw->multiplier);
+
+		for (i = SENSOR_REG_VALUE; i < SENSOR_REG_MAX; i++)
+			dev_debug(dev, "sensor reg[%d]: %x: %zu\n",
+					i, raw->regs[i].regoff,
+					raw->regs[i].size);
+
+		sensor = opae_zmalloc(sizeof(*sensor));
+		if (!sensor) {
+			ret = -EINVAL;
+			goto free_sensor;
+		}
+
+		if (max10_add_sensor(raw, sensor)) {
+			ret = -EINVAL;
+			opae_free(sensor);
+			goto free_sensor;
+		}
+
+		if (sensor->flags & OPAE_SENSOR_VALID)
+			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
+		else
+			opae_free(sensor);
+
+		opae_free(raw);
+	}
+
+	return 0;
+
+free_sensor:
+	if (raw)
+		opae_free(raw);
+	max10_sensor_uinit();
+	return ret;
+}
+
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect)
@@ -235,6 +509,9 @@ struct intel_max10_device *
 	}
 	dev_info(dev, "FPGA loaded from %s Image\n", val ? "User" : "Factory");
 
+
+	max10_sensor_init(dev);
+
 	return dev;
 
 spi_tran_fail:
@@ -253,6 +530,8 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
+	max10_sensor_uinit();
+
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 
diff --git a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
index a52b63e..90bf098 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_intel_max10.h
@@ -103,4 +103,60 @@ struct intel_max10_device *
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
+#define SENSOR_REG_VALUE 0x0
+#define SENSOR_REG_HIGH_WARN 0x1
+#define SENSOR_REG_HIGH_FATAL 0x2
+#define SENSOR_REG_LOW_WARN 0x3
+#define SENSOR_REG_LOW_FATAL 0x4
+#define SENSOR_REG_HYSTERESIS 0x5
+#define SENSOR_REG_MAX 0x6
+
+static const char * const sensor_reg_name[] = {
+	"value",
+	"high_warn",
+	"high_fatal",
+	"low_warn",
+	"low_fatal",
+	"hysteresis",
+};
+
+struct sensor_reg {
+	unsigned int regoff;
+	size_t size;
+};
+
+struct raw_sensor_info {
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int multiplier;
+	struct sensor_reg regs[SENSOR_REG_MAX];
+};
+
+#define OPAE_SENSOR_VALID 0x1
+#define OPAE_SENSOR_HIGH_WARN_VALID 0x2
+#define OPAE_SENSOR_HIGH_FATAL_VALID 0x4
+#define OPAE_SENSOR_LOW_WARN_VALID 0x8
+#define OPAE_SENSOR_LOW_FATAL_VALID 0x10
+#define OPAE_SENSOR_HYSTERESIS_VALID 0x20
+
+struct opae_sensor_info {
+	TAILQ_ENTRY(opae_sensor_info) node;
+	const char *name;
+	const char *type;
+	unsigned int id;
+	unsigned int high_fatal;
+	unsigned int high_warn;
+	unsigned int low_fatal;
+	unsigned int low_warn;
+	unsigned int hysteresis;
+	unsigned int multiplier;
+	unsigned int flags;
+	unsigned int value;
+	unsigned int value_reg;
+};
+
 #endif
-- 
1.8.3.1

[dpdk-dev] [PATCH 08/12] raw/ifpga_rawdev/base: introducing sensor APIs

[Rosen Xu]

From: Tianfei Zhang <tianfei.zhang@intel.com>

Introducing sensor APIs to PMD driver for PAC N3000 card.

Those sensor APIs:
1. opae_mgr_for_each_sensor()
2. opae_mgr_get_sensor_by_name()
3. opae_mgr_get_sensor_by_id()
4. opae_mgr_get_sensor_value_by_name()
5. opae_mgr_get_sensor_value_by_id()
6. opae_mgr_get_sensor_value()

Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_api.c         |  10 ++
 drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h |   3 +
 drivers/raw/ifpga_rawdev/base/ifpga_fme.c         |  21 ++++
 drivers/raw/ifpga_rawdev/base/opae_hw_api.c       | 115 ++++++++++++++++++++++
 drivers/raw/ifpga_rawdev/base/opae_hw_api.h       |  16 +++
 5 files changed, 165 insertions(+)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_api.c b/drivers/raw/ifpga_rawdev/base/ifpga_api.c
index 7ae626d..33d1da3 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_api.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_api.c
@@ -209,9 +209,19 @@ static int ifpga_mgr_get_eth_group_region_info(struct opae_manager *mgr,
 	return 0;
 }
 
+static int ifpga_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct ifpga_fme_hw *fme = mgr->data;
+
+	return fme_mgr_get_sensor_value(fme, sensor, value);
+}
+
 struct opae_manager_ops ifpga_mgr_ops = {
 	.flash = ifpga_mgr_flash,
 	.get_eth_group_region_info = ifpga_mgr_get_eth_group_region_info,
+	.get_sensor_value = ifpga_mgr_get_sensor_value,
 };
 
 static int ifpga_mgr_read_mac_rom(struct opae_manager *mgr, int offset,
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h
index e243d42..2b1309b 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h
@@ -218,4 +218,7 @@ int fme_mgr_get_retimer_info(struct ifpga_fme_hw *fme,
 		struct opae_retimer_info *info);
 int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 		struct opae_retimer_status *status);
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 #endif /* _IFPGA_FEATURE_DEV_H_ */
diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_fme.c b/drivers/raw/ifpga_rawdev/base/ifpga_fme.c
index 2b447fd..794ca09 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_fme.c
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_fme.c
@@ -1300,3 +1300,24 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 
 	return 0;
 }
+
+int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	struct intel_max10_device *dev;
+
+	dev = (struct intel_max10_device *)fme->max10_dev;
+	if (!dev)
+		return -ENODEV;
+
+	if (max10_reg_read(sensor->value_reg, value)) {
+		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
+				__func__, sensor->value_reg);
+		return -EINVAL;
+	}
+
+	*value *= sensor->multiplier;
+
+	return 0;
+}
diff --git a/drivers/raw/ifpga_rawdev/base/opae_hw_api.c b/drivers/raw/ifpga_rawdev/base/opae_hw_api.c
index 8964e79..d0e66d6 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_hw_api.c
+++ b/drivers/raw/ifpga_rawdev/base/opae_hw_api.c
@@ -575,3 +575,118 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
 
 	return -ENOENT;
 }
+
+/**
+ * opae_manager_get_sensor_by_id - get sensor device
+ * @id: the id of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_id(unsigned int id)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (sensor->id == id)
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_by_name - get sensor device
+ * @name: the name of the sensor
+ *
+ * Return: the pointer of the opae_sensor_info
+ */
+struct opae_sensor_info *
+opae_mgr_get_sensor_by_name(const char *name)
+{
+	struct opae_sensor_info *sensor;
+
+	opae_mgr_for_each_sensor(sensor)
+		if (!strcmp(sensor->name, name))
+			return sensor;
+
+	return NULL;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_name - find the sensor by name and read out
+ * the value
+ * @mgr: opae_manager for sensor.
+ * @name: the name of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_name(name);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value_by_id - find the sensor by id and readout the
+ * value
+ * @mgr: opae_manager for sensor
+ * @id: the id of the sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value)
+{
+	struct opae_sensor_info *sensor;
+
+	if (!mgr)
+		return -EINVAL;
+
+	sensor = opae_mgr_get_sensor_by_id(id);
+	if (!sensor)
+		return -ENODEV;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
+
+/**
+ * opae_manager_get_sensor_value - get the current
+ * sensor value
+ * @mgr: opae_manager for sensor
+ * @sensor: opae_sensor_info for sensor
+ * @value: the readout sensor value
+ *
+ * Return: 0 on success, otherwise error code
+ */
+int
+opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value)
+{
+	if (!mgr || !sensor)
+		return -EINVAL;
+
+	if (mgr->ops && mgr->ops->get_sensor_value)
+		return mgr->ops->get_sensor_value(mgr, sensor, value);
+
+	return -ENOENT;
+}
diff --git a/drivers/raw/ifpga_rawdev/base/opae_hw_api.h b/drivers/raw/ifpga_rawdev/base/opae_hw_api.h
index 63405a4..0d7be01 100644
--- a/drivers/raw/ifpga_rawdev/base/opae_hw_api.h
+++ b/drivers/raw/ifpga_rawdev/base/opae_hw_api.h
@@ -48,6 +48,9 @@ struct opae_manager_ops {
 		     u32 size, u64 *status);
 	int (*get_eth_group_region_info)(struct opae_manager *mgr,
 			struct opae_eth_group_region_info *info);
+	int (*get_sensor_value)(struct opae_manager *mgr,
+			struct opae_sensor_info *sensor,
+			unsigned int *value);
 };
 
 /* networking management ops in FME */
@@ -69,6 +72,10 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
+extern struct opae_sensor_list opae_sensor_list;
+#define opae_mgr_for_each_sensor(sensor) \
+	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+
 /* OPAE Manager APIs */
 struct opae_manager *
 opae_manager_alloc(const char *name, struct opae_manager_ops *ops,
@@ -78,6 +85,15 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
+		const char *name, unsigned int *value);
+int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
+		unsigned int id, unsigned int *value);
+int opae_mgr_get_sensor_value(struct opae_manager *mgr,
+		struct opae_sensor_info *sensor,
+		unsigned int *value);
 
 /* OPAE Bridge Data Structure */
 struct opae_bridge_ops;
-- 
1.8.3.1

[dpdk-dev] [PATCH 09/12] raw/ifpga_rawdev/base: update SEU register definition

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Update the SEU registser definition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga_rawdev/base/ifpga_defines.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
index b450cb1..8993cc6 100644
--- a/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
+++ b/drivers/raw/ifpga_rawdev/base/ifpga_defines.h
@@ -1122,7 +1122,9 @@ struct feature_fme_ras_catfaterror {
 			u8  therm_catast_err:1;
 			/* Injected Catastrophic Error */
 			u8  injected_catast_err:1;
-			u64 rsvd:52;
+			/* SEU error on BMC */
+			u8  bmc_seu_catast_err:1;
+			u64 rsvd:51;
 		};
 	};
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH 10/12] raw/ifpga_rawdev: add SEU error handler

[Rosen Xu]

From: Tianfei zhang <tianfei.zhang@intel.com>

Add SEU interrupt support for FPGA.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/raw/ifpga_rawdev/ifpga_rawdev.c | 246 ++++++++++++++++++++++++++++++++
 1 file changed, 246 insertions(+)

diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
index fef89e6..7121884 100644
--- a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
@@ -28,6 +28,8 @@
 #include <rte_bus_vdev.h>
 
 #include "base/opae_hw_api.h"
+#include "base/opae_ifpga_hw_api.h"
+#include "base/ifpga_api.h"
 #include "rte_rawdev.h"
 #include "rte_rawdev_pmd.h"
 #include "rte_bus_ifpga.h"
@@ -606,6 +608,237 @@
 };
 
 static int
+ifpga_get_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 *val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	if (opae_manager_ifpga_get_prop(mgr, &prop))
+		return -EINVAL;
+
+	*val = prop.data;
+
+	return 0;
+}
+
+static int
+ifpga_set_fme_error_prop(struct opae_manager *mgr,
+		u64 prop_id, u64 val)
+{
+	struct feature_prop prop;
+
+	prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
+	prop.prop_id = prop_id;
+
+	prop.data = val;
+
+	if (opae_manager_ifpga_set_prop(mgr, &prop))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_read_seu_emr(struct opae_manager *mgr)
+{
+	struct feature_prop prop;
+	u64 val;
+	int ret;
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%lx\n", val);
+
+	ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", prop.data);
+
+	return 0;
+}
+
+static int fme_clear_warning_intr(struct opae_manager *mgr)
+{
+	u64 val;
+
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
+		return -EINVAL;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+	if ((val & 0x40) != 0)
+		IFPGA_RAWDEV_PMD_INFO("clean not done\n");
+
+	return 0;
+}
+
+static int
+fme_err_handle_error0(struct opae_manager *mgr)
+{
+	struct feature_fme_error0 fme_error0;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
+		return -EINVAL;
+
+	fme_error0.csr = val;
+
+	if (fme_error0.fabric_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
+	else if (fme_error0.fabfifo_overflow)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
+	else if (fme_error0.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
+	else if (fme_error0.pcie0cdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
+	else if (fme_error0.cvlcdc_parity_err)
+		IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
+	else if (fme_error0.fpgaseuerr) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred\n");
+	}
+
+	/* clean the errors */
+	if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+fme_err_handle_catfatal_error(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_catfaterror fme_catfatal;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	fme_catfatal.csr = val;
+
+	if (fme_catfatal.cci_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
+	else if (fme_catfatal.fabric_fatal_err)
+		IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
+	else if (fme_catfatal.pcie_poison_err)
+		IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
+	else if (fme_catfatal.inject_fata_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
+	else if (fme_catfatal.crc_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
+	else if (fme_catfatal.injected_catast_err)
+		IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
+	else if (fme_catfatal.bmc_seu_catast_err) {
+		fme_err_read_seu_emr(mgr);
+		rte_panic("SEU error occurred in BMC\n");
+	}
+
+	return 0;
+}
+
+static int
+fme_err_handle_nonfaterror(struct opae_manager *mgr)
+{
+	struct feature_fme_ras_nonfaterror nonfaterr;
+	u64 val;
+
+	if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
+		return -EINVAL;
+
+	nonfaterr.csr = val;
+
+	if (nonfaterr.temp_thresh_ap1)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
+	else if (nonfaterr.temp_thresh_ap2)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
+	else if (nonfaterr.pcie_error)
+		IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
+	else if (nonfaterr.portfatal_error)
+		IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
+	else if (nonfaterr.proc_hot)
+		IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
+	else if (nonfaterr.afu_acc_mode_err)
+		IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
+	else if (nonfaterr.injected_nonfata_err) {
+		IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
+		fme_clear_warning_intr(mgr);
+	} else if (nonfaterr.temp_thresh_AP6)
+		IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
+	else if (nonfaterr.power_thresh_AP1)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
+	else if (nonfaterr.power_thresh_AP2)
+		IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
+	else if (nonfaterr.mbp_err)
+		IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
+
+	return 0;
+}
+
+static void
+fme_interrupt_handler(void *param)
+{
+	struct opae_manager *mgr = (struct opae_manager *)param;
+
+	IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
+
+	fme_err_handle_error0(mgr);
+	fme_err_handle_nonfaterror(mgr);
+	fme_err_handle_catfatal_error(mgr);
+}
+
+static struct rte_intr_handle fme_intr_handle;
+
+static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
+{
+	int ret;
+	struct fpga_fme_err_irq_set err_irq_set;
+
+	fme_intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX;
+
+	ret = rte_intr_efd_enable(&fme_intr_handle, 1);
+	if (ret)
+		return -EINVAL;
+
+	fme_intr_handle.fd = fme_intr_handle.efds[0];
+
+	IFPGA_RAWDEV_PMD_DEBUG("vfio_dev_fd=%d, efd=%d, fd=%d\n",
+			fme_intr_handle.vfio_dev_fd,
+			fme_intr_handle.efds[0], fme_intr_handle.fd);
+
+	err_irq_set.evtfd = fme_intr_handle.efds[0];
+	ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
+	if (ret)
+		return -EINVAL;
+
+	/* register FME interrupt using DPDK API */
+	ret = rte_intr_callback_register(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+	if (ret)
+		return -EINVAL;
+
+	IFPGA_RAWDEV_PMD_INFO("success register fme interrupt\n");
+
+	return 0;
+}
+
+static int
+ifpga_unregister_fme_interrupt(struct opae_manager *mgr)
+{
+	rte_intr_efd_disable(&fme_intr_handle);
+
+	return rte_intr_callback_unregister(&fme_intr_handle,
+			fme_interrupt_handler,
+			(void *)mgr);
+}
+
+static int
 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
 			int socket_id)
 {
@@ -653,6 +886,7 @@
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
 	/* create a opae_adapter based on above device data */
@@ -678,6 +912,10 @@
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
+	ret = ifpga_register_fme_interrupt(mgr);
+	if (ret)
+		goto free_adapter_data;
+
 	return ret;
 
 free_adapter_data:
@@ -697,6 +935,7 @@
 	struct rte_rawdev *rawdev;
 	char name[RTE_RAWDEV_NAME_MAX_LEN];
 	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
 
 	if (!pci_dev) {
 		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
@@ -721,6 +960,13 @@
 	if (!adapter)
 		return -ENODEV;
 
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	if (ifpga_unregister_fme_interrupt(mgr))
+		return -EINVAL;
+
 	opae_adapter_data_free(adapter->data);
 	opae_adapter_free(adapter);
 
-- 
1.8.3.1

[dpdk-dev] [PATCH 11/12] raw/ifpga_rawdev: add PCIe BDF devices tree scan

[Rosen Xu]

Add PCIe BDF devices tree scan for ipn3ke.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/raw/ifpga_rawdev/ifpga_rawdev.c | 522 +++++++++++++++++++++++++++++++-
 drivers/raw/ifpga_rawdev/ifpga_rawdev.h |  16 +
 2 files changed, 532 insertions(+), 6 deletions(-)

diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
index 7121884..30375fa 100644
--- a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
@@ -8,6 +8,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/epoll.h>
 #include <rte_log.h>
 #include <rte_bus.h>
 #include <rte_eal_memconfig.h>
@@ -18,7 +20,7 @@
 #include <rte_bus_pci.h>
 #include <rte_kvargs.h>
 #include <rte_alarm.h>
-
+#include <rte_interrupts.h>
 #include <rte_errno.h>
 #include <rte_per_lcore.h>
 #include <rte_memory.h>
@@ -26,6 +28,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_bus_vdev.h>
+#include <rte_string_fns.h>
 
 #include "base/opae_hw_api.h"
 #include "base/opae_ifpga_hw_api.h"
@@ -38,6 +41,12 @@
 #include "ifpga_rawdev.h"
 #include "ipn3ke_rawdev_api.h"
 
+#define RTE_PCI_EXT_CAP_ID_ERR    0x01    /* Advanced Error Reporting */
+#define RTE_PCI_CFG_SPACE_SIZE    256
+#define RTE_PCI_CFG_SPACE_EXP_SIZE    4096
+#define RTE_PCI_EXT_CAP_ID(header)    (int)(header & 0x0000ffff)
+#define RTE_PCI_EXT_CAP_NEXT(header)    ((header >> 20) & 0xffc)
+
 int ifpga_rawdev_logtype;
 
 #define PCI_VENDOR_ID_INTEL          0x8086
@@ -65,6 +74,481 @@
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
+
+static int ifpga_monitor_start;
+static pthread_t ifpga_monitor_start_thread;
+
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
+static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev);
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap);
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap);
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	unsigned int i;
+
+	if (rawdev == NULL)
+		return NULL;
+
+	for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+		dev = &ifpga_rawdevices[i];
+		if (dev->rawdev == rawdev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static inline uint8_t
+ifpga_rawdev_find_free_device_index(void)
+{
+	uint16_t dev_id;
+
+	for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
+		if (ifpga_rawdevices[dev_id].rawdev == NULL)
+			return dev_id;
+	}
+
+	return IFPGA_RAWDEV_NUM;
+}
+static struct ifpga_rawdev *
+ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
+{
+	struct ifpga_rawdev *dev;
+	uint16_t dev_id;
+
+	dev = ifpga_rawdev_get(rawdev);
+	if (dev != NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
+		return NULL;
+	}
+
+	dev_id = ifpga_rawdev_find_free_device_index();
+	if (dev_id == IFPGA_RAWDEV_NUM) {
+		IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
+		return NULL;
+	}
+
+	dev = &ifpga_rawdevices[dev_id];
+	dev->rawdev = rawdev;
+	dev->dev_id = dev_id;
+
+	return dev;
+}
+
+static int ifpga_pci_find_next_ext_capability(unsigned int fd,
+int start, int cap)
+{
+	uint32_t header;
+	int ttl;
+	int pos = RTE_PCI_CFG_SPACE_SIZE;
+	int ret;
+
+	/* minimum 8 bytes per capability */
+	ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
+
+	if (start)
+		pos = start;
+	ret = pread(fd, &header, sizeof(header), pos);
+	if (ret == -1)
+		return -1;
+
+	/*
+	 * If we have no capabilities, this is indicated by cap ID,
+	 * cap version and next pointer all being 0.
+	 */
+	if (header == 0)
+		return 0;
+
+	while (ttl-- > 0) {
+		if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
+			return pos;
+
+		pos = RTE_PCI_EXT_CAP_NEXT(header);
+		if (pos < RTE_PCI_CFG_SPACE_SIZE)
+			break;
+		ret = pread(fd, &header, sizeof(header), pos);
+		if (ret == -1)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int ifpga_pci_find_ext_capability(unsigned int fd, int cap)
+{
+	return ifpga_pci_find_next_ext_capability(fd, 0, cap);
+}
+
+static int ifpga_get_dev_vendor_id(const char *bdf,
+	uint32_t *dev_id, uint32_t *vendor_id)
+{
+	int fd;
+	char path[1024];
+	int ret;
+	uint32_t header;
+
+	strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+	strlcat(path, bdf, sizeof(path));
+	strlcat(path, "/config", sizeof(path));
+	fd = open(path, O_RDWR);
+	if (fd < 0)
+		return -1;
+	ret = pread(fd, &header, sizeof(header), 0);
+	if (ret == -1) {
+		close(fd);
+		return -1;
+	}
+	(*vendor_id) = header & 0xffff;
+	(*dev_id) = (header >> 16) & 0xffff;
+	close(fd);
+
+	return 0;
+}
+static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
+	const char *bdf)
+{
+	char path[1024] = "/sys/bus/pci/devices/0000:";
+	char link[1024], link1[1024];
+	char dir[1024] = "/sys/devices/";
+	char *c;
+	int ret;
+	char sub_brg_bdf[4][16];
+	int point;
+	DIR *dp = NULL;
+	struct dirent *entry;
+	int i, j;
+
+	unsigned int dom, bus, dev;
+	int func;
+	uint32_t dev_id, vendor_id;
+
+	strlcat(path, bdf, sizeof(path));
+	memset(link, 0, sizeof(link));
+	memset(link1, 0, sizeof(link1));
+	ret = readlink(path, link, (sizeof(link)-1));
+	if (ret == -1)
+		return -1;
+	strlcpy(link1, link, sizeof(link1));
+	memset(ifpga_dev->parent_bdf, 0, 16);
+	point = strlen(link);
+	if (point < 39)
+		return -1;
+	point -= 39;
+	link[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
+
+	point = strlen(link1);
+	if (point < 26)
+		return -1;
+	point -= 26;
+	link1[point] = 0;
+	if (point < 12)
+		return -1;
+	point -= 12;
+	c = strchr(link1, 'p');
+	if (!c)
+		return -1;
+	strlcat(dir, c, sizeof(dir));
+
+	//scan folder
+	dp = opendir(dir);
+	if (dp == NULL)
+		return -1;
+	i = 0;
+	while ((entry = readdir(dp)) != NULL) {
+		if (i >= 4)
+			break;
+		if (entry->d_name[0] == '.')
+			continue;
+		if (strlen(entry->d_name) > 12)
+			continue;
+		if (sscanf(entry->d_name, "%x:%x:%x.%d",
+			&dom, &bus, &dev, &func) < 4)
+			continue;
+		else {
+			strlcpy(sub_brg_bdf[i],
+				entry->d_name,
+				sizeof(sub_brg_bdf[i]));
+			i++;
+		}
+	}
+	closedir(dp);
+
+	//get fpga and fvl
+	j = 0;
+	for (i = 0; i < 4; i++) {
+		strlcpy(link, dir, sizeof(link));
+		strlcat(link, "/", sizeof(link));
+		strlcat(link, sub_brg_bdf[i], sizeof(link));
+		dp = opendir(link);
+		if (dp == NULL)
+			return -1;
+		while ((entry = readdir(dp)) != NULL) {
+			if (j >= 8)
+				break;
+			if (entry->d_name[0] == '.')
+				continue;
+
+			if (strlen(entry->d_name) > 12)
+				continue;
+			if (sscanf(entry->d_name, "%x:%x:%x.%d",
+				&dom, &bus, &dev, &func) < 4)
+				continue;
+			else {
+				if (ifpga_get_dev_vendor_id(entry->d_name,
+					&dev_id, &vendor_id))
+					continue;
+				if (vendor_id == 0x8086 &&
+					(dev_id == 0x0CF8 ||
+					dev_id == 0x0D58 ||
+					dev_id == 0x1580)) {
+					strlcpy(ifpga_dev->fvl_bdf[j],
+						entry->d_name,
+						sizeof(ifpga_dev->fvl_bdf[j]));
+					j++;
+				}
+			}
+		}
+		closedir(dp);
+	}
+
+	return 0;
+}
+
+#define HIGH_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
+	 (value > (_sens)->high_fatal))
+
+#define HIGH_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
+	 (value > (_sens)->high_warn))
+
+#define LOW_FATAL(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
+	 (value > (_sens)->low_fatal))
+
+#define LOW_WARN(_sens, value)\
+	(((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
+	 (value > (_sens)->low_warn))
+
+static int
+ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
+	       bool *gsd_start)
+{
+	struct opae_adapter *adapter;
+	struct opae_manager *mgr;
+	struct opae_sensor_info *sensor;
+	unsigned int value;
+	int ret;
+
+	adapter = ifpga_rawdev_get_priv(raw_dev);
+	if (!adapter)
+		return -ENODEV;
+
+	mgr = opae_adapter_get_mgr(adapter);
+	if (!mgr)
+		return -ENODEV;
+
+	opae_mgr_for_each_sensor(sensor) {
+		if (!sensor)
+			goto fail;
+
+		if (!(sensor->flags & OPAE_SENSOR_VALID))
+			goto fail;
+
+		/* we monitor 2 sensors */
+		if (!strcmp(sensor->name, "Board Temperature") ||
+				!strcmp(sensor->name, "FPGA Die Temperature")) {
+
+			ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
+			if (ret)
+				goto fail;
+
+			if (value == 0xdeadbeef) {
+				printf("sensor is invalid, value=0x%x\n",
+					value);
+				continue;
+			}
+
+			printf("read sensor:%s data: %d, high_warn:%d, high_fatal:%d\n",
+					sensor->name, value, sensor->high_warn,
+					sensor->high_fatal);
+
+			if (HIGH_WARN(sensor, value) ||
+				LOW_WARN(sensor, value)) {
+				printf("sensor %s reach the warn theshold, value:%d\n",
+					sensor->name, value);
+				*gsd_start = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+fail:
+	return -EFAULT;
+}
+
+static int set_surprise_link_check_aer(struct ifpga_rawdev *ifpga_rdev)
+{
+	struct rte_rawdev *rdev;
+	int fd = -1;
+	char path[1024];
+	int pos;
+	int ret;
+	uint32_t data;
+	bool enable = 0;
+	uint32_t aer_new0, aer_new1;
+
+	if (!ifpga_rdev) {
+		printf("\n device does not exist\n");
+		return -EFAULT;
+	}
+
+	rdev = ifpga_rdev->rawdev;
+	if (ifpga_rdev->aer_enable)
+		return -EFAULT;
+	if (ifpga_monitor_sensor(rdev, &enable))
+		return -EFAULT;
+	if (enable) {
+		IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
+		ifpga_rdev->aer_enable = 1;
+		//get bridge fd
+		strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
+		strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
+		strlcat(path, "/config", sizeof(path));
+		fd = open(path, O_RDWR);
+		if (fd < 0)
+			goto end;
+		pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
+		if (!pos)
+			goto end;
+		//save previout ECAP_AER+0x08
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[0] = data;
+		//save previout ECAP_AER+0x14
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		ifpga_rdev->aer_old[1] = data;
+
+		//set ECAP_AER+0x08 to 0xFFFFFFFF
+		data = 0xffffffff;
+		ret = pwrite(fd, &data, 4, pos+0x08);
+		if (ret == -1)
+			goto end;
+		//set ECAP_AER+0x14 to 0xFFFFFFFF
+		ret = pwrite(fd, &data, 4, pos+0x14);
+		if (ret == -1)
+			goto end;
+
+		//read current ECAP_AER+0x08
+		ret = pread(fd, &data, sizeof(data), pos+0x08);
+		if (ret == -1)
+			goto end;
+		aer_new0 = data;
+		//read current ECAP_AER+0x14
+		ret = pread(fd, &data, sizeof(data), pos+0x14);
+		if (ret == -1)
+			goto end;
+		aer_new1 = data;
+
+		if (fd != -1)
+			close(fd);
+
+		printf(">>>>>>Set AER %x,%x %x,%x\n",
+			ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
+			aer_new0, aer_new1);
+
+		return 1;
+		}
+
+end:
+	if (fd != -1)
+		close(fd);
+	return -EFAULT;
+}
+
+static void *
+ifpga_rawdev_gsd_handle(__rte_unused void *param)
+{
+	struct ifpga_rawdev *ifpga_rdev;
+	int i;
+	int gsd_enable, ret;
+#define MS 1000
+
+	while (1) {
+		for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
+			ifpga_rdev = &ifpga_rawdevices[i];
+			if (ifpga_rdev->rawdev) {
+				printf(">>>>>>Check Device %s\n",
+					ifpga_rdev->rawdev->name);
+				ret = set_surprise_link_check_aer(ifpga_rdev);
+			}
+			if (ret == 1)
+				gsd_enable = 1;
+		}
+
+		if (gsd_enable)
+			rte_exit(EXIT_FAILURE, ">>>>>>Graceful Shutdown\n");
+
+		rte_delay_us(1000 * MS);
+	}
+
+	return NULL;
+}
+
+static int
+ifpga_monitor_start_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 0) {
+		ret = pthread_create(&ifpga_monitor_start_thread,
+			NULL,
+			ifpga_rawdev_gsd_handle, NULL);
+		if (ret) {
+			IFPGA_RAWDEV_PMD_ERR("Fail to create ifpga nonitor thread");
+			return -1;
+		}
+		ifpga_monitor_start = 1;
+	}
+
+	return 0;
+}
+static int
+ifpga_monitor_stop_func(void)
+{
+	int ret;
+
+	if (ifpga_monitor_start == 1) {
+		ret = pthread_cancel(ifpga_monitor_start_thread);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
+
+		ret = pthread_join(ifpga_monitor_start_thread, NULL);
+		if (ret)
+			IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
+
+		ifpga_monitor_start = 0;
+
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 		struct rte_afu_device *afu_dev)
@@ -373,8 +857,9 @@
 	if (ret)
 		return ret;
 
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
-	memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+	rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high,
+		uuid.b + 8, sizeof(u64));
 
 	IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
 		(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
@@ -644,7 +1129,6 @@
 static int
 fme_err_read_seu_emr(struct opae_manager *mgr)
 {
-	struct feature_prop prop;
 	u64 val;
 	int ret;
 
@@ -658,7 +1142,7 @@
 	if (ret)
 		return -EINVAL;
 
-	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", prop.data);
+	IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%lx\n", val);
 
 	return 0;
 }
@@ -844,6 +1328,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	int ret = 0;
 	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *dev = NULL;
 	struct opae_adapter *adapter = NULL;
 	struct opae_manager *mgr = NULL;
 	struct opae_adapter_data_pci *data = NULL;
@@ -857,7 +1342,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(name, 0, sizeof(name));
-	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%02x:%02x.%x",
 		pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
 
 	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
@@ -871,6 +1356,14 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 		goto cleanup;
 	}
 
+	dev = ifpga_rawdev_allocate(rawdev);
+	if (dev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	dev->aer_enable = 0;
+
 	/* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
 	data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
 	if (!data) {
@@ -989,6 +1482,7 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 static int
 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
 {
+	ifpga_monitor_stop_func();
 	return ifpga_rawdev_destroy(pci_dev);
 }
 
@@ -1025,8 +1519,11 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 {
 	struct rte_devargs *devargs;
 	struct rte_kvargs *kvlist = NULL;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga_dev;
 	int port;
 	char *name = NULL;
+	const char *bdf;
 	char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
 	int ret = -1;
 
@@ -1067,6 +1564,19 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 
 	memset(dev_name, 0, sizeof(dev_name));
+	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
+	rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
+	if (!rawdev)
+		goto end;
+	ifpga_dev = ifpga_rawdev_get(rawdev);
+	if (!ifpga_dev)
+		goto end;
+	bdf = name;
+	ifpga_rawdev_fill_info(ifpga_dev, bdf);
+
+	ifpga_monitor_start_func();
+
+	memset(dev_name, 0, sizeof(dev_name));
 	snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
 	port, name);
 
diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.h b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
index e153dba..bd42083 100644
--- a/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
@@ -46,4 +46,20 @@ enum ifpga_rawdev_device_state {
 	return rawdev->dev_private;
 }
 
+#define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
+#define IFPGA_RAWDEV_NUM 32
+
+struct ifpga_rawdev {
+	int dev_id;
+	struct rte_rawdev *rawdev;
+	int aer_enable;
+	int intr_fd[IFPGA_RAWDEV_MSIX_IRQ_NUM+1];
+	uint32_t aer_old[2];
+	char fvl_bdf[8][16];
+	char parent_bdf[16];
+};
+
+struct ifpga_rawdev *
+ifpga_rawdev_get(const struct rte_rawdev *rawdev);
+
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

[dpdk-dev] [PATCH 12/12] net/ipn3ke: remove configuration for i40e port bonding

[Rosen Xu]

The ipn3ke board FPGA and i40e BDF scan has added in ifpga_rawdev,
so it doesn't need to provide configuration for i40e port bonding.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/net/ipn3ke/Makefile             |   2 +
 drivers/net/ipn3ke/ipn3ke_ethdev.c      | 289 ++++----------------------------
 drivers/net/ipn3ke/ipn3ke_representor.c |   7 +-
 3 files changed, 43 insertions(+), 255 deletions(-)

diff --git a/drivers/net/ipn3ke/Makefile b/drivers/net/ipn3ke/Makefile
index 8c3ae37..5478fd9 100644
--- a/drivers/net/ipn3ke/Makefile
+++ b/drivers/net/ipn3ke/Makefile
@@ -19,6 +19,8 @@ CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga_rawdev
+CFLAGS += -I$(RTE_SDK)/drivers/net/i40e
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_ifpga
diff --git a/drivers/net/ipn3ke/ipn3ke_ethdev.c b/drivers/net/ipn3ke/ipn3ke_ethdev.c
index c226d63..363a5f1 100644
--- a/drivers/net/ipn3ke/ipn3ke_ethdev.c
+++ b/drivers/net/ipn3ke/ipn3ke_ethdev.c
@@ -19,6 +19,7 @@
 #include <rte_bus_ifpga.h>
 #include <ifpga_common.h>
 #include <ifpga_logs.h>
+#include <ifpga_rawdev.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -241,7 +242,8 @@
 				"LineSideMACType", &mac_type);
 	hw->retimer.mac_type = (int)mac_type;
 
-	IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n", IPN3KE_READ_REG(hw, 0));
+	hw->acc_tm = 0;
+	hw->acc_flow = 0;
 
 	if (afu_dev->id.uuid.uuid_low == IPN3KE_UUID_VBNG_LOW &&
 		afu_dev->id.uuid.uuid_high == IPN3KE_UUID_VBNG_HIGH) {
@@ -259,6 +261,12 @@
 		/* After reset, wait until init done */
 		if (ipn3ke_vbng_init_done(hw))
 			return -1;
+
+		hw->acc_tm = 1;
+		hw->acc_flow = 1;
+
+		IPN3KE_AFU_PMD_DEBUG("UPL_version is 0x%x\n",
+			IPN3KE_READ_REG(hw, 0));
 	}
 
 	if (hw->retimer.mac_type == IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI) {
@@ -323,9 +331,6 @@
 		hw->flow_hw_enable = 1;
 	}
 
-	hw->acc_tm = 0;
-	hw->acc_flow = 0;
-
 	return 0;
 }
 
@@ -376,7 +381,11 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 {
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct ipn3ke_hw *hw;
-	int i, retval;
+	struct rte_eth_dev *i40e_eth;
+	struct ifpga_rawdev *ifpga_dev;
+	uint16_t port_id;
+	int i, j, retval;
+	char *fvl_bdf;
 
 	/* check if the AFU device has been probed already */
 	/* allocate shared mcp_vswitch structure */
@@ -403,7 +412,12 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 	if (retval)
 		return retval;
 
+	ifpga_dev = ifpga_rawdev_get(hw->rawdev);
+		if (!ifpga_dev)
+			IPN3KE_AFU_PMD_ERR("failed to find ifpga_device.");
+
 	/* probe representor ports */
+	j = 0;
 	for (i = 0; i < hw->port_num; i++) {
 		struct ipn3ke_rpst rpst = {
 			.port_id = i,
@@ -415,6 +429,22 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		snprintf(name, sizeof(name), "net_%s_representor_%d",
 			afu_dev->device.name, i);
 
+		for (; j < 8; j++) {
+			fvl_bdf = ifpga_dev->fvl_bdf[j];
+			retval = rte_eth_dev_get_port_by_name(fvl_bdf,
+				&port_id);
+			if (retval) {
+				continue;
+			} else {
+				i40e_eth = &rte_eth_devices[port_id];
+				rpst.i40e_pf_eth = i40e_eth;
+				rpst.i40e_pf_eth_port_id = port_id;
+
+				j++;
+				break;
+			}
+		}
+
 		retval = rte_eth_dev_create(&afu_dev->device, name,
 			sizeof(struct ipn3ke_rpst), NULL, NULL,
 			ipn3ke_rpst_init, &rpst);
@@ -422,6 +452,7 @@ static int ipn3ke_vswitch_probe(struct rte_afu_device *afu_dev)
 		if (retval)
 			IPN3KE_AFU_PMD_ERR("failed to create ipn3ke representor %s.",
 								name);
+
 	}
 
 	return 0;
@@ -467,254 +498,6 @@ static int ipn3ke_vswitch_remove(struct rte_afu_device *afu_dev)
 
 RTE_PMD_REGISTER_AFU(net_ipn3ke_afu, afu_ipn3ke_driver);
 
-static const char * const valid_args[] = {
-#define IPN3KE_AFU_NAME         "afu"
-		IPN3KE_AFU_NAME,
-#define IPN3KE_FPGA_ACCELERATION_LIST     "fpga_acc"
-		IPN3KE_FPGA_ACCELERATION_LIST,
-#define IPN3KE_I40E_PF_LIST     "i40e_pf"
-		IPN3KE_I40E_PF_LIST,
-		NULL
-};
-
-static int
-ipn3ke_cfg_parse_acc_list(const char *afu_name,
-	const char *acc_list_name)
-{
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = acc_list_name;
-	while (*p_source) {
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-		if (!strcmp(name, "tm") && hw->tm_hw_enable)
-			hw->acc_tm = 1;
-
-		if (!strcmp(name, "flow") && hw->flow_hw_enable)
-			hw->acc_flow = 1;
-
-		if (*p_source == '}')
-			return 0;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_parse_i40e_pf_ethdev(const char *afu_name,
-	const char *pf_name)
-{
-	struct rte_eth_dev *i40e_eth, *rpst_eth;
-	struct rte_afu_device *afu_dev;
-	struct ipn3ke_rpst *rpst;
-	struct ipn3ke_hw *hw;
-	const char *p_source;
-	char *p_start;
-	char name[RTE_ETH_NAME_MAX_LEN];
-	uint16_t port_id;
-	int i;
-	int ret = -1;
-
-	afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-	if (!afu_dev)
-		return -1;
-	hw = afu_dev->shared.data;
-	if (!hw)
-		return -1;
-
-	p_source = pf_name;
-	for (i = 0; i < hw->port_num; i++) {
-		snprintf(name, sizeof(name), "net_%s_representor_%d",
-			afu_name, i);
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		rpst_eth = &rte_eth_devices[port_id];
-		rpst = IPN3KE_DEV_PRIVATE_TO_RPST(rpst_eth);
-
-		while ((*p_source == '{') || (*p_source == '|'))
-			p_source++;
-		p_start = name;
-		while ((*p_source != '|') && (*p_source != '}'))
-			*p_start++ = *p_source++;
-		*p_start = 0;
-
-		ret = rte_eth_dev_get_port_by_name(name, &port_id);
-		if (ret)
-			return -1;
-		i40e_eth = &rte_eth_devices[port_id];
-
-		rpst->i40e_pf_eth = i40e_eth;
-		rpst->i40e_pf_eth_port_id = port_id;
-
-		if ((*p_source == '}') || !(*p_source))
-			break;
-	}
-
-	return 0;
-}
-
-static int
-ipn3ke_cfg_probe(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	char *acc_name = NULL;
-	char *pf_name = NULL;
-	int afu_name_en = 0;
-	int acc_list_en = 0;
-	int pf_list_en = 0;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-			goto end;
-		} else {
-			afu_name_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_FPGA_ACCELERATION_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_FPGA_ACCELERATION_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &acc_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		} else {
-			acc_list_en = 1;
-		}
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_I40E_PF_LIST) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_I40E_PF_LIST,
-				       &rte_ifpga_get_string_arg,
-				       &pf_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_I40E_PF_LIST);
-			goto end;
-		} else {
-			pf_list_en = 1;
-		}
-	}
-
-	if (!afu_name_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_AFU_NAME);
-		goto end;
-	}
-
-	if (!pf_list_en) {
-		IPN3KE_AFU_PMD_ERR("arg %s is mandatory for ipn3ke",
-			  IPN3KE_I40E_PF_LIST);
-		goto end;
-	}
-
-	if (acc_list_en) {
-		ret = ipn3ke_cfg_parse_acc_list(afu_name, acc_name);
-		if (ret) {
-			IPN3KE_AFU_PMD_ERR("arg %s parse error for ipn3ke",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-			goto end;
-		}
-	} else {
-		IPN3KE_AFU_PMD_INFO("arg %s is optional for ipn3ke, using i40e acc",
-			  IPN3KE_FPGA_ACCELERATION_LIST);
-	}
-
-	ret = ipn3ke_cfg_parse_i40e_pf_ethdev(afu_name, pf_name);
-	if (ret)
-		goto end;
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-	if (afu_name)
-		free(afu_name);
-	if (acc_name)
-		free(acc_name);
-
-	return ret;
-}
-
-static int
-ipn3ke_cfg_remove(struct rte_vdev_device *dev)
-{
-	struct rte_devargs *devargs;
-	struct rte_kvargs *kvlist = NULL;
-	char *afu_name = NULL;
-	struct rte_afu_device *afu_dev;
-	int ret = -1;
-
-	devargs = dev->device.devargs;
-
-	kvlist = rte_kvargs_parse(devargs->args, valid_args);
-	if (!kvlist) {
-		IPN3KE_AFU_PMD_ERR("error when parsing param");
-		goto end;
-	}
-
-	if (rte_kvargs_count(kvlist, IPN3KE_AFU_NAME) == 1) {
-		if (rte_kvargs_process(kvlist, IPN3KE_AFU_NAME,
-				       &rte_ifpga_get_string_arg,
-				       &afu_name) < 0) {
-			IPN3KE_AFU_PMD_ERR("error to parse %s",
-				     IPN3KE_AFU_NAME);
-		} else {
-			afu_dev = rte_ifpga_find_afu_by_name(afu_name);
-			if (!afu_dev)
-				goto end;
-			ret = ipn3ke_vswitch_remove(afu_dev);
-		}
-	} else {
-		IPN3KE_AFU_PMD_ERR("Remove ipn3ke_cfg %p error", dev);
-	}
-
-end:
-	if (kvlist)
-		rte_kvargs_free(kvlist);
-
-	return ret;
-}
-
-static struct rte_vdev_driver ipn3ke_cfg_driver = {
-	.probe = ipn3ke_cfg_probe,
-	.remove = ipn3ke_cfg_remove,
-};
-
-RTE_PMD_REGISTER_VDEV(ipn3ke_cfg, ipn3ke_cfg_driver);
-RTE_PMD_REGISTER_PARAM_STRING(ipn3ke_cfg,
-	"afu=<string> "
-	"fpga_acc=<string>"
-	"i40e_pf=<string>");
-
 RTE_INIT(ipn3ke_afu_init_log)
 {
 	ipn3ke_afu_logtype = rte_log_register("pmd.afu.ipn3ke");
diff --git a/drivers/net/ipn3ke/ipn3ke_representor.c b/drivers/net/ipn3ke/ipn3ke_representor.c
index 8300cc3..a4ee460 100644
--- a/drivers/net/ipn3ke/ipn3ke_representor.c
+++ b/drivers/net/ipn3ke/ipn3ke_representor.c
@@ -20,6 +20,7 @@
 #include <rte_rawdev_pmd.h>
 #include <rte_bus_ifpga.h>
 #include <ifpga_logs.h>
+#include <rte_pmd_i40e.h>
 
 #include "ipn3ke_rawdev_api.h"
 #include "ipn3ke_flow.h"
@@ -2906,8 +2907,10 @@ static uint16_t ipn3ke_rpst_recv_pkts(__rte_unused void *rx_q,
 	rpst->switch_domain_id = representor_param->switch_domain_id;
 	rpst->port_id = representor_param->port_id;
 	rpst->hw = representor_param->hw;
-	rpst->i40e_pf_eth = NULL;
-	rpst->i40e_pf_eth_port_id = 0xFFFF;
+	rpst->i40e_pf_eth = representor_param->i40e_pf_eth;
+	rpst->i40e_pf_eth_port_id = representor_param->i40e_pf_eth_port_id;
+	if (rpst->i40e_pf_eth)
+		i40e_set_switch_dev(rpst->i40e_pf_eth, rpst->ethdev);
 
 	ethdev->data->mac_addrs = rte_zmalloc("ipn3ke", RTE_ETHER_ADDR_LEN, 0);
 	if (!ethdev->data->mac_addrs) {
-- 
1.8.3.1