[dpdk-dev] [PATCH v11 18/19] raw/ifpga/base: add multiple cards support

[Andy Pei <andy.pei@intel.com>]

From: Tianfei zhang <tianfei.zhang@intel.com>

In PAC N3000 card, there is one MAX10 chip in each card, and
all of the sensors are connected to MAX10 chip. To support multiple
cards in one server, we introducing a sensor device list under
intel_max10_device instead of a global list. On the other hand, we
using separate intel_max10_device instance for each opae_adatper.

Add mutex lock on do_transaction() function for SPI driver to avoid
race condition.

Signed-off-by: Tianfei zhang <tianfei.zhang@intel.com>
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/raw/ifpga/base/ifpga_fme.c            |  40 +++++++---
 drivers/raw/ifpga/base/opae_debug.c           |   3 +
 drivers/raw/ifpga/base/opae_hw_api.c          |  14 ++--
 drivers/raw/ifpga/base/opae_hw_api.h          |  15 ++--
 drivers/raw/ifpga/base/opae_i2c.c             |  44 ++++++++---
 drivers/raw/ifpga/base/opae_i2c.h             |   3 +-
 drivers/raw/ifpga/base/opae_intel_max10.c     | 110 ++++++++++++++------------
 drivers/raw/ifpga/base/opae_intel_max10.h     |  19 +++--
 drivers/raw/ifpga/base/opae_spi.c             |   5 --
 drivers/raw/ifpga/base/opae_spi.h             |   3 +-
 drivers/raw/ifpga/base/opae_spi_transaction.c |  44 +++++++++--
 drivers/raw/ifpga/ifpga_rawdev.c              |  14 ++--
 12 files changed, 205 insertions(+), 109 deletions(-)

diff --git a/drivers/raw/ifpga/base/ifpga_fme.c b/drivers/raw/ifpga/base/ifpga_fme.c
index 799d67d..c31a94c 100644
--- a/drivers/raw/ifpga/base/ifpga_fme.c
+++ b/drivers/raw/ifpga/base/ifpga_fme.c
@@ -839,8 +839,13 @@ static int board_type_to_info(u32 type,
 static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 {
 	struct fme_bitstream_id id;
+	struct ifpga_hw *hw;
 	u32 val;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
 	if (fme_hdr_get_bitstream_id(fme, &id.id))
 		return -EINVAL;
 
@@ -854,7 +859,10 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	fme->board_info.seu = id.seu;
 	fme->board_info.ptp = id.ptp;
 
-	dev_info(fme, "found: board: %s type: %s\n",
+	dev_info(fme, "found: PCI dev: %02x:%02x:%x board: %s type: %s\n",
+			hw->pci_data->bus,
+			hw->pci_data->devid,
+			hw->pci_data->function,
 			board_major_to_string(fme->board_info.major),
 			board_type_to_string(fme->board_info.type));
 
@@ -882,11 +890,11 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 			fme->board_info.nums_of_fvl,
 			fme->board_info.ports_per_fvl);
 
-	if (max10_sys_read(MAX10_BUILD_VER, &val))
+	if (max10_sys_read(fme->max10_dev, MAX10_BUILD_VER, &val))
 		return -EINVAL;
 	fme->board_info.max10_version = val & 0xffffff;
 
-	if (max10_sys_read(NIOS2_FW_VERSION, &val))
+	if (max10_sys_read(fme->max10_dev, NIOS2_FW_VERSION, &val))
 		return -EINVAL;
 	fme->board_info.nios_fw_version = val & 0xffffff;
 
@@ -897,12 +905,12 @@ static int fme_get_board_interface(struct ifpga_fme_hw *fme)
 	return 0;
 }
 
-static int spi_self_checking(void)
+static int spi_self_checking(struct intel_max10_device *dev)
 {
 	u32 val;
 	int ret;
 
-	ret = max10_sys_read(MAX10_TEST_REG, &val);
+	ret = max10_sys_read(dev, MAX10_TEST_REG, &val);
 	if (ret)
 		return -EIO;
 
@@ -937,10 +945,11 @@ static int fme_spi_init(struct ifpga_feature *feature)
 		goto spi_fail;
 	}
 
+
 	fme->max10_dev = max10;
 
 	/* SPI self test */
-	if (spi_self_checking()) {
+	if (spi_self_checking(max10)) {
 		ret = -EIO;
 		goto max10_fail;
 	}
@@ -1041,8 +1050,18 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 	struct ifpga_fme_hw *fme = (struct ifpga_fme_hw *)feature->parent;
 	struct altera_spi_device *spi_master;
 	struct intel_max10_device *max10;
+	struct ifpga_hw *hw;
+	struct opae_manager *mgr;
 	int ret = 0;
 
+	hw = fme->parent;
+	if (!hw)
+		return -ENODEV;
+
+	mgr = hw->adapter->mgr;
+	if (!mgr)
+		return -ENODEV;
+
 	dev_info(fme, "FME SPI Master (NIOS) Init.\n");
 	dev_debug(fme, "FME SPI base addr %p.\n",
 			feature->addr);
@@ -1080,12 +1099,15 @@ static int fme_nios_spi_init(struct ifpga_feature *feature)
 		goto release_dev;
 	}
 
+	max10->bus = hw->pci_data->bus;
+
 	fme_get_board_interface(fme);
 
 	fme->max10_dev = max10;
+	mgr->sensor_list = &max10->opae_sensor_list;
 
 	/* SPI self test */
-	if (spi_self_checking())
+	if (spi_self_checking(max10))
 		goto spi_fail;
 
 	return ret;
@@ -1344,7 +1366,7 @@ int fme_mgr_get_retimer_status(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(PKVL_LINK_STATUS, &val)) {
+	if (max10_sys_read(dev, PKVL_LINK_STATUS, &val)) {
 		dev_err(dev, "%s: read pkvl status fail\n", __func__);
 		return -EINVAL;
 	}
@@ -1372,7 +1394,7 @@ int fme_mgr_get_sensor_value(struct ifpga_fme_hw *fme,
 	if (!dev)
 		return -ENODEV;
 
-	if (max10_sys_read(sensor->value_reg, value)) {
+	if (max10_sys_read(dev, sensor->value_reg, value)) {
 		dev_err(dev, "%s: read sensor value register 0x%x fail\n",
 				__func__, sensor->value_reg);
 		return -EINVAL;
diff --git a/drivers/raw/ifpga/base/opae_debug.c b/drivers/raw/ifpga/base/opae_debug.c
index 88f2d5c..dad3ea3 100644
--- a/drivers/raw/ifpga/base/opae_debug.c
+++ b/drivers/raw/ifpga/base/opae_debug.c
@@ -59,6 +59,9 @@ static void opae_adapter_data_dump(void *data)
 		opae_log("OPAE Adapter Type = PCI\n");
 		opae_log("PCI Device ID: 0x%04x\n", d_pci->device_id);
 		opae_log("PCI Vendor ID: 0x%04x\n", d_pci->vendor_id);
+		opae_log("PCI bus: 0x%04x\n", d_pci->bus);
+		opae_log("PCI devid: 0x%04x\n", d_pci->devid);
+		opae_log("PCI function: 0x%04x\n", d_pci->function);
 
 		for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 			r = &d_pci->region[i];
diff --git a/drivers/raw/ifpga/base/opae_hw_api.c b/drivers/raw/ifpga/base/opae_hw_api.c
index 1ccc967..c969dfe 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.c
+++ b/drivers/raw/ifpga/base/opae_hw_api.c
@@ -583,11 +583,12 @@ int opae_manager_get_retimer_status(struct opae_manager *mgr,
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_id(unsigned int id)
+opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (sensor->id == id)
 			return sensor;
 
@@ -601,11 +602,12 @@ struct opae_sensor_info *
  * Return: the pointer of the opae_sensor_info
  */
 struct opae_sensor_info *
-opae_mgr_get_sensor_by_name(const char *name)
+opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name)
 {
 	struct opae_sensor_info *sensor;
 
-	opae_mgr_for_each_sensor(sensor)
+	opae_mgr_for_each_sensor(mgr, sensor)
 		if (!strcmp(sensor->name, name))
 			return sensor;
 
@@ -630,7 +632,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_name(name);
+	sensor = opae_mgr_get_sensor_by_name(mgr, name);
 	if (!sensor)
 		return -ENODEV;
 
@@ -658,7 +660,7 @@ struct opae_sensor_info *
 	if (!mgr)
 		return -EINVAL;
 
-	sensor = opae_mgr_get_sensor_by_id(id);
+	sensor = opae_mgr_get_sensor_by_id(mgr, id);
 	if (!sensor)
 		return -ENODEV;
 
diff --git a/drivers/raw/ifpga/base/opae_hw_api.h b/drivers/raw/ifpga/base/opae_hw_api.h
index b78fbd5..cdf369f 100644
--- a/drivers/raw/ifpga/base/opae_hw_api.h
+++ b/drivers/raw/ifpga/base/opae_hw_api.h
@@ -40,6 +40,7 @@ struct opae_manager {
 	struct opae_adapter *adapter;
 	struct opae_manager_ops *ops;
 	struct opae_manager_networking_ops *network_ops;
+	struct opae_sensor_list *sensor_list;
 	void *data;
 };
 
@@ -75,9 +76,8 @@ struct opae_manager_networking_ops {
 			struct opae_retimer_status *status);
 };
 
-extern struct opae_sensor_list opae_sensor_list;
-#define opae_mgr_for_each_sensor(sensor) \
-	TAILQ_FOREACH(sensor, &opae_sensor_list, node)
+#define opae_mgr_for_each_sensor(mgr, sensor) \
+	TAILQ_FOREACH(sensor, mgr->sensor_list, node)
 
 /* OPAE Manager APIs */
 struct opae_manager *
@@ -88,8 +88,10 @@ int opae_manager_flash(struct opae_manager *mgr, int acc_id, const char *buf,
 		       u32 size, u64 *status);
 int opae_manager_get_eth_group_region_info(struct opae_manager *mgr,
 		u8 group_id, struct opae_eth_group_region_info *info);
-struct opae_sensor_info *opae_mgr_get_sensor_by_name(const char *name);
-struct opae_sensor_info *opae_mgr_get_sensor_by_id(unsigned int id);
+struct opae_sensor_info *opae_mgr_get_sensor_by_name(struct opae_manager *mgr,
+		const char *name);
+struct opae_sensor_info *opae_mgr_get_sensor_by_id(struct opae_manager *mgr,
+		unsigned int id);
 int opae_mgr_get_sensor_value_by_name(struct opae_manager *mgr,
 		const char *name, unsigned int *value);
 int opae_mgr_get_sensor_value_by_id(struct opae_manager *mgr,
@@ -241,6 +243,9 @@ struct opae_adapter_data_pci {
 	enum opae_adapter_type type;
 	u16 device_id;
 	u16 vendor_id;
+	u16 bus; /*Device bus for PCI */
+	u16 devid; /* Device ID */
+	u16 function; /* Device function */
 	struct opae_reg_region region[PCI_MAX_RESOURCE];
 	int vfio_dev_fd;  /* VFIO device file descriptor */
 };
diff --git a/drivers/raw/ifpga/base/opae_i2c.c b/drivers/raw/ifpga/base/opae_i2c.c
index f8bc247..846d751 100644
--- a/drivers/raw/ifpga/base/opae_i2c.c
+++ b/drivers/raw/ifpga/base/opae_i2c.c
@@ -28,6 +28,9 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 {
 	u8 msgbuf[2];
 	int i = 0;
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
 
 	if (flags & I2C_FLAG_ADDR16)
 		msgbuf[i++] = offset >> 8;
@@ -49,10 +52,16 @@ int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		},
 	};
 
-	if (!dev->xfer)
-		return -ENODEV;
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
+
+	ret = i2c_transfer(dev, msg, 2);
 
-	return i2c_transfer(dev, msg, 2);
+exit:
+	pthread_mutex_unlock(&dev->lock);
+	return ret;
 }
 
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
@@ -63,12 +72,18 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	int ret;
 	int i = 0;
 
-	if (!dev->xfer)
-		return -ENODEV;
+	pthread_mutex_lock(&dev->lock);
+
+	if (!dev->xfer) {
+		ret = -ENODEV;
+		goto exit;
+	}
 
 	buf = opae_malloc(I2C_MAX_OFFSET_LEN + len);
-	if (!buf)
-		return -ENOMEM;
+	if (!buf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
 
 	msg.addr = slave_addr;
 	msg.flags = 0;
@@ -84,6 +99,8 @@ int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 	ret = i2c_transfer(dev, &msg, 1);
 
 	opae_free(buf);
+exit:
+	pthread_mutex_unlock(&dev->lock);
 	return ret;
 }
 
@@ -477,14 +494,19 @@ struct altera_i2c_dev *altera_i2c_probe(void *base)
 	dev->i2c_clk = dev->i2c_param.ref_clk * 1000000;
 	dev->xfer = altera_i2c_xfer;
 
+	if (pthread_mutex_init(&dev->lock, NULL))
+		return NULL;
+
 	altera_i2c_hardware_init(dev);
 
 	return dev;
 }
 
-int altera_i2c_remove(struct altera_i2c_dev *dev)
+void altera_i2c_remove(struct altera_i2c_dev *dev)
 {
-	altera_i2c_disable(dev);
-
-	return 0;
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
+		altera_i2c_disable(dev);
+		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/base/opae_i2c.h b/drivers/raw/ifpga/base/opae_i2c.h
index 8890c8f..266e127 100644
--- a/drivers/raw/ifpga/base/opae_i2c.h
+++ b/drivers/raw/ifpga/base/opae_i2c.h
@@ -93,6 +93,7 @@ struct altera_i2c_dev {
 	u32 isr_mask;
 	u8 *buf;
 	int (*xfer)(struct altera_i2c_dev *dev, struct i2c_msg *msg, int num);
+	pthread_mutex_t lock;
 };
 
 /**
@@ -114,7 +115,7 @@ enum i2c_msg_flags {
 };
 
 struct altera_i2c_dev *altera_i2c_probe(void *base);
-int altera_i2c_remove(struct altera_i2c_dev *dev);
+void altera_i2c_remove(struct altera_i2c_dev *dev);
 int i2c_read(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
 		u32 offset, u8 *buf, u32 count);
 int i2c_write(struct altera_i2c_dev *dev, int flags, unsigned int slave_addr,
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.c b/drivers/raw/ifpga/base/opae_intel_max10.c
index 748ab56..8e23ca1 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.c
+++ b/drivers/raw/ifpga/base/opae_intel_max10.c
@@ -5,45 +5,50 @@
 #include "opae_intel_max10.h"
 #include <libfdt.h>
 
-static struct intel_max10_device *g_max10;
-
-struct opae_sensor_list opae_sensor_list =
-	TAILQ_HEAD_INITIALIZER(opae_sensor_list);
-
-int max10_reg_read(unsigned int reg, unsigned int *val)
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_read(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x\n", __func__, dev->bus, reg);
+
+	return spi_transaction_read(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)val);
 }
 
-int max10_reg_write(unsigned int reg, unsigned int val)
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val)
 {
 	unsigned int tmp = val;
 
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return spi_transaction_write(g_max10->spi_tran_dev,
+	dev_debug(dev, "%s: bus:0x%x, reg:0x%x, val:0x%x\n", __func__,
+			dev->bus, reg, val);
+
+	return spi_transaction_write(dev->spi_tran_dev,
 			reg, 4, (unsigned char *)&tmp);
 }
 
-int max10_sys_read(unsigned int offset, unsigned int *val)
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_read(g_max10->base + offset, val);
+
+	return max10_reg_read(dev, dev->base + offset, val);
 }
 
-int max10_sys_write(unsigned int offset, unsigned int val)
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val)
 {
-	if (!g_max10)
+	if (!dev)
 		return -ENODEV;
 
-	return max10_reg_write(g_max10->base + offset, val);
+	return max10_reg_write(dev, dev->base + offset, val);
 }
 
 static struct max10_compatible_id max10_id_table[] = {
@@ -86,8 +91,8 @@ static void max10_check_capability(struct intel_max10_device *max10)
 		max10->flags |= MAX10_FLAGS_MAC_CACHE;
 }
 
-static int altera_nor_flash_read(u32 offset,
-		void *buffer, u32 len)
+static int altera_nor_flash_read(struct intel_max10_device *dev,
+	u32 offset, void *buffer, u32 len)
 {
 	int word_len;
 	int i;
@@ -95,13 +100,13 @@ static int altera_nor_flash_read(u32 offset,
 	unsigned int value;
 	int ret;
 
-	if (!buffer || len <= 0)
+	if (!dev || !buffer || len <= 0)
 		return -ENODEV;
 
 	word_len = len/4;
 
 	for (i = 0; i < word_len; i++) {
-		ret = max10_reg_read(offset + i*4,
+		ret = max10_reg_read(dev, offset + i*4,
 				&value);
 		if (ret)
 			return -EBUSY;
@@ -112,12 +117,12 @@ static int altera_nor_flash_read(u32 offset,
 	return 0;
 }
 
-static int enable_nor_flash(bool on)
+static int enable_nor_flash(struct intel_max10_device *dev, bool on)
 {
 	unsigned int val = 0;
 	int ret;
 
-	ret = max10_sys_read(RSU_REG, &val);
+	ret = max10_sys_read(dev, RSU_REG, &val);
 	if (ret) {
 		dev_err(NULL "enabling flash error\n");
 		return ret;
@@ -128,7 +133,7 @@ static int enable_nor_flash(bool on)
 	else
 		val &= ~RSU_ENABLE;
 
-	return max10_sys_write(RSU_REG, val);
+	return max10_sys_write(dev, RSU_REG, val);
 }
 
 static int init_max10_device_table(struct intel_max10_device *max10)
@@ -140,7 +145,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	u32 dt_size, dt_addr, val;
 	int ret;
 
-	ret = max10_sys_read(DT_AVAIL_REG, &val);
+	ret = max10_sys_read(max10, DT_AVAIL_REG, &val);
 	if (ret) {
 		dev_err(max10 "cannot read DT_AVAIL_REG\n");
 		return ret;
@@ -151,19 +156,19 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		return -EINVAL;
 	}
 
-	ret = max10_sys_read(DT_BASE_ADDR_REG, &dt_addr);
+	ret = max10_sys_read(max10, DT_BASE_ADDR_REG, &dt_addr);
 	if (ret) {
 		dev_info(max10 "cannot get base addr of device table\n");
 		return ret;
 	}
 
-	ret = enable_nor_flash(true);
+	ret = enable_nor_flash(max10, true);
 	if (ret) {
 		dev_err(max10 "fail to enable flash\n");
 		return ret;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, &hdr, sizeof(hdr));
+	ret = altera_nor_flash_read(max10, dt_addr, &hdr, sizeof(hdr));
 	if (ret) {
 		dev_err(max10 "read fdt header fail\n");
 		goto done;
@@ -188,7 +193,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 		goto done;
 	}
 
-	ret = altera_nor_flash_read(dt_addr, fdt_root, dt_size);
+	ret = altera_nor_flash_read(max10, dt_addr, fdt_root, dt_size);
 	if (ret) {
 		dev_err(max10 "cannot read device table\n");
 		goto done;
@@ -207,7 +212,7 @@ static int init_max10_device_table(struct intel_max10_device *max10)
 	max10->fdt_root = fdt_root;
 
 done:
-	ret = enable_nor_flash(false);
+	ret = enable_nor_flash(max10, false);
 
 	if (ret && fdt_root)
 		opae_free(fdt_root);
@@ -298,12 +303,12 @@ static int fdt_get_named_reg(const void *fdt, int node, const char *name,
 	return fdt_get_reg(fdt, node, idx, start, size);
 }
 
-static void max10_sensor_uinit(void)
+static void max10_sensor_uinit(struct intel_max10_device *dev)
 {
 	struct opae_sensor_info *info;
 
-	TAILQ_FOREACH(info, &opae_sensor_list, node) {
-		TAILQ_REMOVE(&opae_sensor_list, info, node);
+	TAILQ_FOREACH(info, &dev->opae_sensor_list, node) {
+		TAILQ_REMOVE(&dev->opae_sensor_list, info, node);
 		opae_free(info);
 	}
 }
@@ -313,8 +318,8 @@ static bool sensor_reg_valid(struct sensor_reg *reg)
 	return !!reg->size;
 }
 
-static int max10_add_sensor(struct raw_sensor_info *info,
-		struct opae_sensor_info *sensor)
+static int max10_add_sensor(struct intel_max10_device *dev,
+	struct raw_sensor_info *info, struct opae_sensor_info *sensor)
 {
 	int i;
 	int ret = 0;
@@ -332,12 +337,15 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		if (!sensor_reg_valid(&info->regs[i]))
 			continue;
 
-		ret = max10_sys_read(info->regs[i].regoff, &val);
+		ret = max10_sys_read(dev, info->regs[i].regoff, &val);
 		if (ret)
 			break;
 
-		if (val == 0xdeadbeef)
+		if (val == 0xdeadbeef) {
+			dev_debug(dev, "%s: sensor:%s invalid 0x%x at:%d\n",
+				__func__, sensor->name, val, i);
 			continue;
+		}
 
 		val *= info->multiplier;
 
@@ -458,7 +466,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 		num = fdt_getprop(fdt_root, offset, "multiplier", NULL);
 		raw->multiplier = num ? fdt32_to_cpu(*num) : 1;
 
-		dev_info(dev, "found sensor from DTB: %s: %s: %u: %u\n",
+		dev_debug(dev, "found sensor from DTB: %s: %s: %u: %u\n",
 				raw->name, raw->type,
 				raw->id, raw->multiplier);
 
@@ -473,15 +481,16 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 			goto free_sensor;
 		}
 
-		if (max10_add_sensor(raw, sensor)) {
+		if (max10_add_sensor(dev, raw, sensor)) {
 			ret = -EINVAL;
 			opae_free(sensor);
 			goto free_sensor;
 		}
 
-		if (sensor->flags & OPAE_SENSOR_VALID)
-			TAILQ_INSERT_TAIL(&opae_sensor_list, sensor, node);
-		else
+		if (sensor->flags & OPAE_SENSOR_VALID) {
+			TAILQ_INSERT_TAIL(&dev->opae_sensor_list, sensor, node);
+			dev_info(dev, "found valid sensor: %s\n", sensor->name);
+		} else
 			opae_free(sensor);
 
 		opae_free(raw);
@@ -492,7 +501,7 @@ static int max10_add_sensor(struct raw_sensor_info *info,
 free_sensor:
 	if (raw)
 		opae_free(raw);
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 	return ret;
 }
 
@@ -500,7 +509,7 @@ static int check_max10_version(struct intel_max10_device *dev)
 {
 	unsigned int v;
 
-	if (!max10_reg_read(MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
+	if (!max10_reg_read(dev, MAX10_SEC_BASE_ADDR + MAX10_BUILD_VER,
 				&v)) {
 		if (v != 0xffffffff) {
 			dev_info(dev, "secure MAX10 detected\n");
@@ -565,6 +574,8 @@ struct intel_max10_device *
 	if (!dev)
 		return NULL;
 
+	TAILQ_INIT(&dev->opae_sensor_list);
+
 	dev->spi_master = spi;
 
 	dev->spi_tran_dev = spi_transaction_init(spi, chipselect);
@@ -573,9 +584,6 @@ struct intel_max10_device *
 		goto free_dev;
 	}
 
-	/* set the max10 device firstly */
-	g_max10 = dev;
-
 	/* check the max10 version */
 	ret = check_max10_version(dev);
 	if (ret) {
@@ -601,7 +609,7 @@ struct intel_max10_device *
 	}
 
 	/* read FPGA loading information */
-	ret = max10_sys_read(FPGA_PAGE_INFO, &val);
+	ret = max10_sys_read(dev, FPGA_PAGE_INFO, &val);
 	if (ret) {
 		dev_err(dev, "fail to get FPGA loading info\n");
 		goto release_max10_hw;
@@ -611,14 +619,13 @@ struct intel_max10_device *
 	return dev;
 
 release_max10_hw:
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 free_dtb:
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
 free_dev:
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return NULL;
@@ -629,7 +636,7 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (!dev)
 		return 0;
 
-	max10_sensor_uinit();
+	max10_sensor_uinit(dev);
 
 	if (dev->spi_tran_dev)
 		spi_transaction_remove(dev->spi_tran_dev);
@@ -637,7 +644,6 @@ int intel_max10_device_remove(struct intel_max10_device *dev)
 	if (dev->fdt_root)
 		opae_free(dev->fdt_root);
 
-	g_max10 = NULL;
 	opae_free(dev);
 
 	return 0;
diff --git a/drivers/raw/ifpga/base/opae_intel_max10.h b/drivers/raw/ifpga/base/opae_intel_max10.h
index e632941..123cdc4 100644
--- a/drivers/raw/ifpga/base/opae_intel_max10.h
+++ b/drivers/raw/ifpga/base/opae_intel_max10.h
@@ -26,6 +26,9 @@ struct max10_compatible_id {
 #define MAX10_FLAGS_SECURE		BIT(6)
 #define MAX10_FLAGS_MAC_CACHE		BIT(7)
 
+/** List of opae sensors */
+TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
+
 struct intel_max10_device {
 	unsigned int flags; /*max10 hardware capability*/
 	struct altera_spi_device *spi_master;
@@ -33,6 +36,8 @@ struct intel_max10_device {
 	struct max10_compatible_id *id; /*max10 compatible*/
 	char *fdt_root;
 	unsigned int base; /* max10 base address */
+	u16 bus;
+	struct opae_sensor_list opae_sensor_list;
 };
 
 /* retimer speed */
@@ -136,17 +141,19 @@ struct opae_retimer_status {
 
 #define DFT_MAX_SIZE		0x7e0000
 
-int max10_reg_read(unsigned int reg, unsigned int *val);
-int max10_reg_write(unsigned int reg, unsigned int val);
-int max10_sys_read(unsigned int offset, unsigned int *val);
-int max10_sys_write(unsigned int offset, unsigned int val);
+int max10_reg_read(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int *val);
+int max10_reg_write(struct intel_max10_device *dev,
+	unsigned int reg, unsigned int val);
+int max10_sys_read(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int *val);
+int max10_sys_write(struct intel_max10_device *dev,
+	unsigned int offset, unsigned int val);
 struct intel_max10_device *
 intel_max10_device_probe(struct altera_spi_device *spi,
 		int chipselect);
 int intel_max10_device_remove(struct intel_max10_device *dev);
 
-/** List of opae sensors */
-TAILQ_HEAD(opae_sensor_list, opae_sensor_info);
 
 #define SENSOR_REG_VALUE 0x0
 #define SENSOR_REG_HIGH_WARN 0x1
diff --git a/drivers/raw/ifpga/base/opae_spi.c b/drivers/raw/ifpga/base/opae_spi.c
index cc52782..bfdc83e 100644
--- a/drivers/raw/ifpga/base/opae_spi.c
+++ b/drivers/raw/ifpga/base/opae_spi.c
@@ -181,7 +181,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 	u32 rxd;
 	unsigned int tx_data;
 	u32 status;
-	int retry = 0;
 	int ret;
 
 	while (count < dev->len) {
@@ -194,10 +193,6 @@ static int spi_txrx(struct altera_spi_device *dev)
 				return -EIO;
 			if (status & ALTERA_SPI_STATUS_RRDY_MSK)
 				break;
-			if (retry++ > SPI_MAX_RETRY) {
-				dev_err(dev, "%s, read timeout\n", __func__);
-				return -EBUSY;
-			}
 		}
 
 		ret = spi_reg_read(dev, ALTERA_SPI_RXDATA, &rxd);
diff --git a/drivers/raw/ifpga/base/opae_spi.h b/drivers/raw/ifpga/base/opae_spi.h
index 6355deb..d20a4c3 100644
--- a/drivers/raw/ifpga/base/opae_spi.h
+++ b/drivers/raw/ifpga/base/opae_spi.h
@@ -38,7 +38,7 @@
 #define SPI_WRITE 0x20
 #define WRITE_DATA_MASK GENMASK_ULL(31, 0)
 
-#define SPI_MAX_RETRY 100000
+#define SPI_MAX_RETRY 1000000
 
 #define TYPE_SPI 0
 #define TYPE_NIOS_SPI 1
@@ -102,6 +102,7 @@ struct spi_transaction_dev {
 	struct altera_spi_device *dev;
 	int chipselect;
 	struct spi_tran_buffer *buffer;
+	pthread_mutex_t lock;
 };
 
 struct spi_tran_header {
diff --git a/drivers/raw/ifpga/base/opae_spi_transaction.c b/drivers/raw/ifpga/base/opae_spi_transaction.c
index 06ca625..013efee 100644
--- a/drivers/raw/ifpga/base/opae_spi_transaction.c
+++ b/drivers/raw/ifpga/base/opae_spi_transaction.c
@@ -154,6 +154,8 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	unsigned int rx_len = 0;
 	int retry = 0;
 	int spi_flags;
+	unsigned long timeout = msecs_to_timer_cycles(1000);
+	unsigned long ticks;
 	unsigned int resp_max_len = 2 * resp_len;
 
 	print_buffer("before bytes:", send_data, send_len);
@@ -207,8 +209,11 @@ static int byte_to_core_convert(struct spi_transaction_dev *dev,
 	if (ret != SPI_FOUND_EOP) {
 		tx_buffer = NULL;
 		tx_len = 0;
-		if (retry++ > 10) {
-			dev_err(NULL, "cannot found valid data from SPI\n");
+		ticks = rte_get_timer_cycles();
+		if (time_after(ticks, timeout) &&
+				retry++ > SPI_MAX_RETRY) {
+			dev_err(NULL, "Have retry %d, found invalid packet data\n",
+				retry);
 			return -EBUSY;
 		}
 
@@ -427,23 +432,36 @@ static int do_transaction(struct spi_transaction_dev *dev, unsigned int addr,
 int spi_transaction_read(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_READ : SPI_TRAN_NON_SEQ_READ);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 int spi_transaction_write(struct spi_transaction_dev *dev, unsigned int addr,
 		unsigned int size, unsigned char *data)
 {
-	return do_transaction(dev, addr, size, data,
+	int ret;
+
+	pthread_mutex_lock(&dev->lock);
+	ret = do_transaction(dev, addr, size, data,
 			(size > SPI_REG_BYTES) ?
 			SPI_TRAN_SEQ_WRITE : SPI_TRAN_NON_SEQ_WRITE);
+	pthread_mutex_unlock(&dev->lock);
+
+	return ret;
 }
 
 struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 		int chipselect)
 {
 	struct spi_transaction_dev *spi_tran_dev;
+	int ret;
 
 	spi_tran_dev = opae_malloc(sizeof(struct spi_transaction_dev));
 	if (!spi_tran_dev)
@@ -453,18 +471,28 @@ struct spi_transaction_dev *spi_transaction_init(struct altera_spi_device *dev,
 	spi_tran_dev->chipselect = chipselect;
 
 	spi_tran_dev->buffer = opae_malloc(sizeof(struct spi_tran_buffer));
-	if (!spi_tran_dev->buffer) {
-		opae_free(spi_tran_dev);
-		return NULL;
+	if (!spi_tran_dev->buffer)
+		goto err;
+
+	ret = pthread_mutex_init(&spi_tran_dev->lock, NULL);
+	if (ret) {
+		dev_err(spi_tran_dev, "fail to init mutex lock\n");
+		goto err;
 	}
 
 	return spi_tran_dev;
+
+err:
+	opae_free(spi_tran_dev);
+	return NULL;
 }
 
 void spi_transaction_remove(struct spi_transaction_dev *dev)
 {
 	if (dev && dev->buffer)
 		opae_free(dev->buffer);
-	if (dev)
+	if (dev) {
+		pthread_mutex_destroy(&dev->lock);
 		opae_free(dev);
+	}
 }
diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index e87af66..7253644 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -360,7 +360,7 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 	if (!mgr)
 		return -ENODEV;
 
-	opae_mgr_for_each_sensor(sensor) {
+	opae_mgr_for_each_sensor(mgr, sensor) {
 		if (!(sensor->flags & OPAE_SENSOR_VALID))
 			goto fail;
 
@@ -369,8 +369,8 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 			goto fail;
 
 		if (value == 0xdeadbeef) {
-			IFPGA_RAWDEV_PMD_ERR("sensor %s is invalid value %x\n",
-					sensor->name, value);
+			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value %x\n",
+					raw_dev->dev_id, sensor->name, value);
 			continue;
 		}
 
@@ -393,8 +393,9 @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev,
 		/* monitor 12V AUX sensor */
 		if (!strcmp(sensor->name, "12V AUX Voltage")) {
 			if (value < AUX_VOLTAGE_WARN) {
-				IFPGA_RAWDEV_PMD_INFO("%s reach theshold %d\n",
-						sensor->name, value);
+				IFPGA_RAWDEV_PMD_INFO(
+					"%s reach theshold %d mV\n",
+					sensor->name, value);
 				*gsd_start = true;
 				break;
 			}
@@ -1433,6 +1434,9 @@ static int ifpga_register_fme_interrupt(struct opae_manager *mgr)
 	}
 	data->device_id = pci_dev->id.device_id;
 	data->vendor_id = pci_dev->id.vendor_id;
+	data->bus = pci_dev->addr.bus;
+	data->devid = pci_dev->addr.devid;
+	data->function = pci_dev->addr.function;
 	data->vfio_dev_fd = pci_dev->intr_handle.vfio_dev_fd;
 
 	adapter = rawdev->dev_private;
-- 
1.8.3.1