[dpdk-dev] [PATCH v4 06/12] virtio, qtest: Add pci device initialization function to qtest utils

[Tetsuya Mukawa <mukawa@igel.co.jp>]

The patch adds general pci device initialization functionality to
qtest utils. It initializes pci devices using qtest messaging.

Signed-off-by: Tetsuya Mukawa <mukawa@igel.co.jp>
---
 drivers/net/virtio/qtest_utils.c | 349 ++++++++++++++++++++++++++++++++++++++-
 drivers/net/virtio/qtest_utils.h | 114 ++++++++++++-
 2 files changed, 461 insertions(+), 2 deletions(-)

diff --git a/drivers/net/virtio/qtest_utils.c b/drivers/net/virtio/qtest_utils.c
index f4cd6af..000c7e8 100644
--- a/drivers/net/virtio/qtest_utils.c
+++ b/drivers/net/virtio/qtest_utils.c
@@ -43,6 +43,10 @@
 #include "virtio_ethdev.h"
 #include "qtest_utils.h"
 
+#define PCI_CONFIG_ADDR(_bus, _device, _function, _offset) ( \
+	(1 << 31) | ((_bus) & 0xff) << 16 | ((_device) & 0x1f) << 11 | \
+	((_function) & 0x7) << 8 | ((_offset) & 0xfc))
+
 union qtest_pipefds {
 	struct {
 		int pipefd[2];
@@ -57,6 +61,8 @@ struct qtest_session {
 	int qtest_socket;
 	pthread_mutex_t qtest_session_lock;
 
+	struct qtest_pci_device_list head;
+
 	pthread_t event_th;
 	int event_th_started;
 	char *evq;
@@ -195,6 +201,119 @@ qtest_raw_write(struct qtest_session *s, uint64_t addr, uint32_t val, char type)
 }
 
 /*
+ * qtest_pci_inX/outX are used for accessing PCI configuration space.
+ * The functions are implemented based on PCI configuration space
+ * specification.
+ * Accroding to the spec, access size of read()/write() should be 4 bytes.
+ */
+static int
+qtest_pci_inb(struct qtest_session *s, uint8_t bus, uint8_t device,
+		uint8_t function, uint8_t offset)
+{
+	uint32_t tmp;
+
+	tmp = PCI_CONFIG_ADDR(bus, device, function, offset);
+
+	if (pthread_mutex_lock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	qtest_raw_out(s, 0xcf8, tmp, 'l');
+	tmp = qtest_raw_in(s, 0xcfc, 'l');
+
+	if (pthread_mutex_unlock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot unlock mutex\n");
+
+	return (tmp >> ((offset & 0x3) * 8)) & 0xff;
+}
+
+static uint32_t
+qtest_pci_inl(struct qtest_session *s, uint8_t bus, uint8_t device,
+		uint8_t function, uint8_t offset)
+{
+	uint32_t tmp;
+
+	tmp = PCI_CONFIG_ADDR(bus, device, function, offset);
+
+	if (pthread_mutex_lock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	qtest_raw_out(s, 0xcf8, tmp, 'l');
+	tmp = qtest_raw_in(s, 0xcfc, 'l');
+
+	if (pthread_mutex_unlock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot unlock mutex\n");
+
+	return tmp;
+}
+
+static void
+qtest_pci_outl(struct qtest_session *s, uint8_t bus, uint8_t device,
+		uint8_t function, uint8_t offset, uint32_t value)
+{
+	uint32_t tmp;
+
+	tmp = PCI_CONFIG_ADDR(bus, device, function, offset);
+
+	if (pthread_mutex_lock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	qtest_raw_out(s, 0xcf8, tmp, 'l');
+	qtest_raw_out(s, 0xcfc, value, 'l');
+
+	if (pthread_mutex_unlock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot unlock mutex\n");
+}
+
+static uint64_t
+qtest_pci_inq(struct qtest_session *s, uint8_t bus, uint8_t device,
+		uint8_t function, uint8_t offset)
+{
+	uint32_t tmp;
+	uint64_t val;
+
+	tmp = PCI_CONFIG_ADDR(bus, device, function, offset);
+
+	if (pthread_mutex_lock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	qtest_raw_out(s, 0xcf8, tmp, 'l');
+	val = (uint64_t)qtest_raw_in(s, 0xcfc, 'l');
+
+	tmp = PCI_CONFIG_ADDR(bus, device, function, offset + 4);
+
+	qtest_raw_out(s, 0xcf8, tmp, 'l');
+	val |= (uint64_t)qtest_raw_in(s, 0xcfc, 'l') << 32;
+
+	if (pthread_mutex_unlock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot unlock mutex\n");
+
+	return val;
+}
+
+static void
+qtest_pci_outq(struct qtest_session *s, uint8_t bus, uint8_t device,
+		uint8_t function, uint8_t offset, uint64_t value)
+{
+	uint32_t tmp;
+
+	tmp = PCI_CONFIG_ADDR(bus, device, function, offset);
+
+	if (pthread_mutex_lock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	qtest_raw_out(s, 0xcf8, tmp, 'l');
+	qtest_raw_out(s, 0xcfc, (uint32_t)(value & 0xffffffff), 'l');
+
+	tmp = PCI_CONFIG_ADDR(bus, device, function, offset + 4);
+
+	qtest_raw_out(s, 0xcf8, tmp, 'l');
+	qtest_raw_out(s, 0xcfc, (uint32_t)(value >> 32), 'l');
+
+	if (pthread_mutex_unlock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot unlock mutex\n");
+}
+
+/*
  * qtest_in/out are used for accessing ioport of qemu guest.
  * qtest_read/write are used for accessing memory of qemu guest.
  */
@@ -254,6 +373,18 @@ qtest_write(struct qtest_session *s, uint64_t addr, uint64_t val, char type)
 		rte_panic("Cannot lock mutex\n");
 }
 
+static struct qtest_pci_device *
+qtest_find_device(struct qtest_session *s, const char *name)
+{
+	struct qtest_pci_device *dev;
+
+	TAILQ_FOREACH(dev, &s->head, next) {
+		if (strcmp(dev->name, name) == 0)
+			return dev;
+	}
+	return NULL;
+}
+
 static void
 qtest_event_send(struct qtest_session *s, char *buf)
 {
@@ -382,6 +513,208 @@ qtest_event_handler(void *data) {
 	return NULL;
 }
 
+/*
+ * Common initialization of PCI device.
+ * To know detail, see pci specification.
+ */
+int
+qtest_init_pci_device(struct qtest_session *s, struct qtest_pci_device *dev)
+{
+	uint8_t i, bus, device;
+	uint32_t val;
+	uint64_t val64;
+
+	bus = dev->bus_addr;
+	device = dev->device_addr;
+
+	PMD_DRV_LOG(INFO,
+		"Find %s on virtual PCI bus: %04x:%02x:00.0\n",
+		dev->name, bus, device);
+
+	/* Check header type */
+	val = qtest_pci_inb(s, bus, device, 0, PCI_HEADER_TYPE);
+	if (val != PCI_HEADER_TYPE_NORMAL) {
+		PMD_DRV_LOG(ERR, "Unexpected header type %d\n", val);
+		return -1;
+	}
+
+	/* Check BAR type */
+	for (i = 0; i < NB_BAR; i++) {
+		val = qtest_pci_inl(s, bus, device, 0, dev->bar[i].addr);
+
+		switch (dev->bar[i].type) {
+		case QTEST_PCI_BAR_IO:
+			if ((val & 0x1) != PCI_BASE_ADDRESS_SPACE_IO)
+				dev->bar[i].type = QTEST_PCI_BAR_DISABLE;
+			break;
+		case QTEST_PCI_BAR_MEMORY_UNDER_1MB:
+			if ((val & 0x1) != PCI_BASE_ADDRESS_SPACE_MEMORY)
+				dev->bar[i].type = QTEST_PCI_BAR_DISABLE;
+			if ((val & 0x6) != PCI_BASE_ADDRESS_MEM_TYPE_1M)
+				dev->bar[i].type = QTEST_PCI_BAR_DISABLE;
+			break;
+		case QTEST_PCI_BAR_MEMORY_32:
+			if ((val & 0x1) != PCI_BASE_ADDRESS_SPACE_MEMORY)
+				dev->bar[i].type = QTEST_PCI_BAR_DISABLE;
+			if ((val & 0x6) != PCI_BASE_ADDRESS_MEM_TYPE_32)
+				dev->bar[i].type = QTEST_PCI_BAR_DISABLE;
+			break;
+		case QTEST_PCI_BAR_MEMORY_64:
+
+			if ((val & 0x1) != PCI_BASE_ADDRESS_SPACE_MEMORY)
+				dev->bar[i].type = QTEST_PCI_BAR_DISABLE;
+			if ((val & 0x6) != PCI_BASE_ADDRESS_MEM_TYPE_64)
+				dev->bar[i].type = QTEST_PCI_BAR_DISABLE;
+			break;
+		case QTEST_PCI_BAR_DISABLE:
+			break;
+		}
+	}
+
+	/* Enable device */
+	val = qtest_pci_inl(s, bus, device, 0, PCI_COMMAND);
+	val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER;
+	qtest_pci_outl(s, bus, device, 0, PCI_COMMAND, val);
+
+	/* Calculate BAR size */
+	for (i = 0; i < NB_BAR; i++) {
+		switch (dev->bar[i].type) {
+		case QTEST_PCI_BAR_IO:
+		case QTEST_PCI_BAR_MEMORY_UNDER_1MB:
+		case QTEST_PCI_BAR_MEMORY_32:
+			qtest_pci_outl(s, bus, device, 0,
+					dev->bar[i].addr, 0xffffffff);
+			val = qtest_pci_inl(s, bus, device,
+					0, dev->bar[i].addr);
+			dev->bar[i].region_size = ~(val & 0xfffffff0) + 1;
+			break;
+		case QTEST_PCI_BAR_MEMORY_64:
+			qtest_pci_outq(s, bus, device, 0,
+					dev->bar[i].addr, 0xffffffffffffffff);
+			val64 = qtest_pci_inq(s, bus, device,
+					0, dev->bar[i].addr);
+			dev->bar[i].region_size =
+					~(val64 & 0xfffffffffffffff0) + 1;
+			break;
+		case QTEST_PCI_BAR_DISABLE:
+			break;
+		}
+	}
+
+	/* Set BAR region */
+	for (i = 0; i < NB_BAR; i++) {
+		switch (dev->bar[i].type) {
+		case QTEST_PCI_BAR_IO:
+		case QTEST_PCI_BAR_MEMORY_UNDER_1MB:
+		case QTEST_PCI_BAR_MEMORY_32:
+			qtest_pci_outl(s, bus, device, 0, dev->bar[i].addr,
+				dev->bar[i].region_start);
+			PMD_DRV_LOG(INFO, "Set BAR of %s device: 0x%lx - 0x%lx\n",
+				dev->name, dev->bar[i].region_start,
+				dev->bar[i].region_start + dev->bar[i].region_size);
+			break;
+		case QTEST_PCI_BAR_MEMORY_64:
+			qtest_pci_outq(s, bus, device, 0, dev->bar[i].addr,
+				dev->bar[i].region_start);
+			PMD_DRV_LOG(INFO, "Set BAR of %s device: 0x%lx - 0x%lx\n",
+				dev->name, dev->bar[i].region_start,
+				dev->bar[i].region_start + dev->bar[i].region_size);
+			break;
+		case QTEST_PCI_BAR_DISABLE:
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int
+qtest_find_pci_device(struct qtest_session *s, const char *name)
+{
+	struct qtest_pci_device *dev;
+	struct rte_pci_addr *addr;
+	uint32_t val;
+
+	dev = qtest_find_device(s, name);
+	if (dev == NULL)
+		goto not_found;
+
+	addr = &dev->specified_addr;
+	PMD_DRV_LOG(INFO, "PCI address of %s is %04x:%02x:%02x.%02x\n", name,
+			addr->domain, addr->bus, addr->devid, addr->function);
+
+	val = qtest_pci_inl(s, addr->bus, addr->devid, addr->function, 0);
+	if (val == ((uint32_t)dev->device_id << 16 | dev->vendor_id)) {
+		dev->bus_addr = addr->bus;
+		dev->device_addr = addr->devid;
+		return 0;
+	}
+
+not_found:
+	PMD_DRV_LOG(ERR, "%s isn' found\n", name);
+	return -1;
+}
+
+static int
+qtest_init_pci_devices(struct qtest_session *s,
+		struct qtest_pci_device *devices, int devnum)
+{
+	struct qtest_pci_device *dev;
+	int i, ret;
+
+
+	/* Try to find devices */
+	for (i = 0; i < devnum; i++) {
+		ret = qtest_find_pci_device(s, devices[i].name);
+		if (ret < 0)
+			return -1;
+	}
+
+	/* Initialize devices */
+	TAILQ_FOREACH(dev, &s->head, next) {
+		ret = dev->init(s, dev);
+		if (ret != 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void
+qtest_remove_target_devices(struct qtest_session *s)
+{
+	struct qtest_pci_device *dev, *next;
+
+	for (dev = TAILQ_FIRST(&s->head); dev != NULL; dev = next) {
+		next = TAILQ_NEXT(dev, next);
+		TAILQ_REMOVE(&s->head, dev, next);
+		free(dev);
+	}
+}
+
+static int
+qtest_register_target_devices(struct qtest_session *s,
+		struct qtest_pci_device *devices, int devnum)
+{
+	struct qtest_pci_device *device;
+	int i;
+
+	TAILQ_INIT(&s->head);
+
+	for (i = 0; i < devnum; i++) {
+		device = malloc(sizeof(*device));
+		if (device == NULL) {
+			qtest_remove_target_devices(s);
+			return -1;
+		}
+
+		*device = devices[i];
+		TAILQ_INSERT_TAIL(&s->head, device, next);
+	}
+
+	return 0;
+}
+
 static int
 qtest_open_socket(char *path)
 {
@@ -431,11 +764,13 @@ qtest_vdev_uninit(struct qtest_session *s)
 	}
 
 	pthread_mutex_destroy(&s->qtest_session_lock);
+	qtest_remove_target_devices(s);
 	rte_free(s);
 }
 
 struct qtest_session *
-qtest_vdev_init(char *qtest_path)
+qtest_vdev_init(char *qtest_path,
+		struct qtest_pci_device *devices, int devnum)
 {
 	struct qtest_session *s;
 	int ret;
@@ -459,6 +794,12 @@ qtest_vdev_init(char *qtest_path)
 		goto error;
 	}
 
+	ret = qtest_register_target_devices(s, devices, devnum);
+	if (ret != 0) {
+		PMD_DRV_LOG(ERR, "Failed to initialize qtest session\n");
+		goto error;
+	}
+
 	s->qtest_socket = qtest_open_socket(qtest_path);
 	if (s->qtest_socket < 0) {
 		PMD_DRV_LOG(ERR, "Failed to open %s\n", qtest_path);
@@ -472,6 +813,12 @@ qtest_vdev_init(char *qtest_path)
 	}
 	s->event_th_started = 1;
 
+	ret = qtest_init_pci_devices(s, devices, devnum);
+	if (ret != 0) {
+		PMD_DRV_LOG(ERR, "Failed to initialize devices\n");
+		goto error;
+	}
+
 	return s;
 
 error:
diff --git a/drivers/net/virtio/qtest_utils.h b/drivers/net/virtio/qtest_utils.h
index 962fc5c..ba70754 100644
--- a/drivers/net/virtio/qtest_utils.h
+++ b/drivers/net/virtio/qtest_utils.h
@@ -34,16 +34,114 @@
 #ifndef _VIRTIO_QTEST_UTILS_H_
 #define _VIRTIO_QTEST_UTILS_H_
 
+#include <sys/queue.h>
+#include <linux/pci_regs.h>
+
+#define NB_BAR                          6
+
+/*
+ * QTest utilities
+ *
+ * This utility assumes QTest guest will have below 3 pci devices.
+ * - piix3
+ *    It will be used for enabling interrupts from target device.
+ * - ivshmme
+ *    It will be used for enabling shared memory between guest and DPDK PMD.
+ * - target device
+ *    It will be the device DPDK PMD wants to use.
+ *    So far, virtio-net device is the only use case.
+ *
+ * To use the utilities, DPDK PMD needs to define above device information.
+ * Then call qtest_vdev_init().
+ * To handle multiple target devices in one QEMU guest, piix3 handling should
+ * be changed.
+ */
+
+enum qtest_pci_bar_type {
+	QTEST_PCI_BAR_DISABLE = 0,
+	QTEST_PCI_BAR_IO,
+	QTEST_PCI_BAR_MEMORY_UNDER_1MB,
+	QTEST_PCI_BAR_MEMORY_32,
+	QTEST_PCI_BAR_MEMORY_64
+};
+
+/*
+ * A structure used to specify BAR information.
+ *
+ * - type
+ *    Specify type of this device.
+ * - addr
+ *    Specify one of PCI_BASE_ADDRESS_0/../5.
+ * - region_start
+ *    Specify physical address of this device. Because a guest cpu will access
+ *    this device using the address, this address should not be over lapped by
+ *    others.
+ * - region_size
+ *    Will be filled by QTest utility while initializing the device.
+ */
+struct qtest_pci_bar {
+	enum qtest_pci_bar_type type;
+	uint8_t addr;
+	uint64_t region_start;
+	uint64_t region_size;
+};
+
+struct qtest_session;
+
+/*
+ * A structure used to specify pci device information.
+ *
+ * - name
+ *    Specify name of this device.
+ * - device_id
+ *    Specify device id of this device.
+ * - vendor_id
+ *    Specify vendor id of this device.
+ * - bus_addr
+ *    Will be filled by QTest utility.
+ *    It will be bus address of this device.
+ * - device_addr
+ *    Will be filled by QTest utility.
+ *    It will be device address of this device.
+ * - bar
+ *    Specify bar structure for this device.
+ * - specified_addr
+ *    Specify pci address of this device.
+ *    QTest utility will not check any other pci address for this device.
+ *    If it's wrong, device initialization will be failed.
+ * - init
+ *   Specify initialization function.
+ *   If the device is generic device, just specify qtest_init_pci_device().
+ */
+TAILQ_HEAD(qtest_pci_device_list, qtest_pci_device);
+struct qtest_pci_device {
+	TAILQ_ENTRY(qtest_pci_device) next;
+	const char *name;
+	uint16_t device_id;
+	uint16_t vendor_id;
+	uint8_t bus_addr;
+	uint8_t device_addr;
+	struct qtest_pci_bar bar[NB_BAR];
+	struct rte_pci_addr specified_addr;
+	int (*init)(struct qtest_session *s, struct qtest_pci_device *dev);
+};
+
 /**
  * @internal
  * Initialization function of QTest utility.
  *
  * @param qtest_path
  *   Path of qtest socket.
+ * @param devices
+ *   Array of device information. It should contain piix3, ivshmem and target
+ *   device(virtio-net device).
+ * @param devnum
+ *   The number of device information.
  * @return
  *   The pointer to qtest session structure.
  */
-struct qtest_session *qtest_vdev_init(char *qtest_path);
+struct qtest_session *qtest_vdev_init(char *qtest_path,
+		struct qtest_pci_device *devices, int devnum);
 
 /**
  * @internal
@@ -116,4 +214,18 @@ uint32_t qtest_read(struct qtest_session *s, uint64_t addr, char type);
 void qtest_write(struct qtest_session *s, uint64_t addr,
 			uint64_t val, char type);
 
+/**
+ * @internal
+ * Initialization function of general device.
+ *
+ * @param s
+ *   The pointer to qtest session structure.
+ * @param dev
+ *   The pointer of pci device.
+ * @return
+ *   0 on success, negative on error
+ */
+int qtest_init_pci_device(struct qtest_session *s,
+			struct qtest_pci_device *dev);
+
 #endif /* _VIRTIO_QTEST_UTILS_H_ */
-- 
2.1.4