From: Stephen Hemminger <shemminger@vyatta.com>
To: dev@dpdk.org
Subject: [dpdk-dev] [PATCH 5/7] pci: support multiple PCI regions per device
Date: Thu, 30 May 2013 10:12:39 -0700 [thread overview]
Message-ID: <20130530171627.005239011@vyatta.com> (raw)
In-Reply-To: <20130530171234.301927271@vyatta.com>
[-- Attachment #1: eal-pci-multiple.patch --]
[-- Type: text/plain, Size: 14007 bytes --]
---
app/test-pmd/config.c | 2
app/test-pmd/testpmd.h | 8 -
lib/librte_eal/common/eal_common_pci.c | 11 +
lib/librte_eal/common/include/rte_pci.h | 7
lib/librte_eal/linuxapp/eal/eal_pci.c | 244 +++++++++++++++++++-------------
lib/librte_pmd_ixgbe/ixgbe_ethdev.c | 4
6 files changed, 169 insertions(+), 107 deletions(-)
--- a/lib/librte_eal/common/eal_common_pci.c 2013-05-29 08:45:38.000000000 -0700
+++ b/lib/librte_eal/common/eal_common_pci.c 2013-05-29 16:09:36.351108977 -0700
@@ -121,12 +121,19 @@ rte_eal_pci_probe(void)
static int
pci_dump_one_device(struct rte_pci_device *dev)
{
+ int i;
+
printf(PCI_PRI_FMT, dev->addr.domain, dev->addr.bus,
dev->addr.devid, dev->addr.function);
printf(" - vendor:%x device:%x\n", dev->id.vendor_id,
dev->id.device_id);
- printf(" %16.16"PRIx64" %16.16"PRIx64"\n",
- dev->mem_resource.phys_addr, dev->mem_resource.len);
+
+ for (i = 0; i < PCI_MEM_RESOURCE; i++) {
+ printf(" %16.16"PRIx64" %16.16"PRIx64"\n",
+ dev->mem_resource[i].phys_addr,
+ dev->mem_resource[i].len);
+ }
+
return 0;
}
--- a/lib/librte_eal/common/include/rte_pci.h 2013-05-29 09:02:50.000000000 -0700
+++ b/lib/librte_eal/common/include/rte_pci.h 2013-05-29 16:30:28.456968882 -0700
@@ -50,6 +50,7 @@ extern "C" {
#include <sys/queue.h>
#include <stdint.h>
#include <inttypes.h>
+#include <limits.h>
#include <rte_interrupts.h>
TAILQ_HEAD(pci_device_list, rte_pci_device); /**< PCI devices in D-linked Q. */
@@ -81,6 +82,8 @@ struct rte_pci_resource {
/** Maximum number of PCI resources. */
#define PCI_MAX_RESOURCE 7
+/** Maximum number of PCI memory resources. */
+#define PCI_MEM_RESOURCE 5
/**
* A structure describing an ID for a PCI driver. Each driver provides a
@@ -110,10 +113,12 @@ struct rte_pci_device {
TAILQ_ENTRY(rte_pci_device) next; /**< Next probed PCI device. */
struct rte_pci_addr addr; /**< PCI location. */
struct rte_pci_id id; /**< PCI ID. */
- struct rte_pci_resource mem_resource; /**< PCI Memory Resource */
+ struct rte_pci_resource mem_resource[PCI_MEM_RESOURCE];
+ /**< PCI Memory Resource */
struct rte_intr_handle intr_handle; /**< Interrupt handle */
const struct rte_pci_driver *driver; /**< Associated driver */
unsigned int blacklisted:1; /**< Device is blacklisted */
+ char uio_name[PATH_MAX]; /**< Associated UIO device name */
};
/** Any PCI device identifier (vendor, device, ...) */
--- a/lib/librte_eal/linuxapp/eal/eal_pci.c 2013-05-29 09:04:00.000000000 -0700
+++ b/lib/librte_eal/linuxapp/eal/eal_pci.c 2013-05-29 16:37:39.600742359 -0700
@@ -99,7 +99,6 @@ struct uio_resource {
struct rte_pci_addr pci_addr;
void *addr;
- char path[PATH_MAX];
unsigned long size;
unsigned long offset;
};
@@ -212,64 +211,108 @@ pci_uio_bind_device(struct rte_pci_devic
return 0;
}
-/* map a particular resource from a file */
-static void *
-pci_map_resource(struct rte_pci_device *dev, void *requested_addr, const char *devname,
- unsigned long offset, unsigned long size)
+/*
+ * open devname: it can take some time to
+ * appear, so we wait some time before returning an error
+ */
+static int uio_open(const char *devname)
{
- unsigned n;
- int fd;
- void *mapaddr;
+ int n, fd;
- /*
- * open devname, and mmap it: it can take some time to
- * appear, so we wait some time before returning an error
- */
- for (n=0; n<UIO_DEV_WAIT_TIMEOUT*10; n++) {
+ for (n=0; n < UIO_DEV_WAIT_TIMEOUT*10; n++) {
fd = open(devname, O_RDWR);
if (fd >= 0)
- break;
+ return fd;
+
if (errno != ENOENT)
break;
usleep(100000);
}
- if (fd < 0) {
- RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", devname, strerror(errno));
- goto fail;
- }
+ return -1;
+}
+
+/* map a particular resource from a file */
+static void *
+pci_mmap(int fd, void *addr, off_t offset, size_t size)
+{
+ void *mapaddr;
/* Map the PCI memory resource of device */
- mapaddr = mmap(requested_addr, size, PROT_READ | PROT_WRITE,
- MAP_SHARED, fd, offset);
- if (mapaddr == MAP_FAILED ||
- (requested_addr != NULL && mapaddr != requested_addr)) {
- RTE_LOG(ERR, EAL, "%s(): cannot mmap %s: %s\n", __func__,
- devname, strerror(errno));
- close(fd);
- goto fail;
- }
- if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
- /* save fd if in primary process */
- dev->intr_handle.fd = fd;
- dev->intr_handle.type = RTE_INTR_HANDLE_UIO;
- } else {
- /* fd is not needed in slave process, close it */
- dev->intr_handle.fd = -1;
- dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
- close(fd);
+ mapaddr = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_SHARED,
+ fd, offset);
+ if (mapaddr == MAP_FAILED || (addr != NULL && mapaddr != addr)) {
+ RTE_LOG(ERR, EAL, "%s(): cannot mmap %zd@0x%lx: %s\n",
+ __func__, size, offset, strerror(errno));
+ return NULL;
}
RTE_LOG(DEBUG, EAL, "PCI memory mapped at %p\n", mapaddr);
-
return mapaddr;
+}
+
+/* save the mapping details for secondary processes*/
+static int pci_uio_map_save(const struct rte_pci_device *dev, void *mapaddr,
+ unsigned long offset, unsigned long size)
+{
+ struct uio_resource *uio_res;
+
+ uio_res = rte_malloc("UIO_RES", sizeof(*uio_res), 0);
+ if (uio_res == NULL) {
+ RTE_LOG(ERR, EAL, "%s(): cannot store uio mmap details\n",
+ __func__);
+ return -1;
+ }
+
+ uio_res->addr = mapaddr;
+ uio_res->offset = offset;
+ uio_res->size = size;
+ memcpy(&uio_res->pci_addr, &dev->addr, sizeof(uio_res->pci_addr));
+
+ TAILQ_INSERT_TAIL(uio_res_list, uio_res, next);
+ return 0;
+}
+
+static int pci_uio_map_restore(struct rte_pci_device *dev)
+{
+ struct uio_resource *uio_res;
+ int i, fd;
+ void *addr;
-fail:
dev->intr_handle.fd = -1;
dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
- return NULL;
+ fd = uio_open(dev->uio_name);
+ if (fd < 0)
+ return -1;
+
+ TAILQ_FOREACH(uio_res, uio_res_list, next) {
+ /* skip this element if it doesn't match our PCI address */
+ if (memcmp(&uio_res->pci_addr, &dev->addr, sizeof(dev->addr)))
+ continue;
+
+ for (i = 0; i < PCI_MEM_RESOURCE; i++) {
+ if (dev->mem_resource[i].len == 0)
+ continue;
+
+ addr = pci_mmap(fd, uio_res->addr,
+ uio_res->offset, uio_res->size);
+ if (addr != uio_res->addr) {
+ RTE_LOG(ERR, EAL, "Cannot mmap device resource\n");
+ close(fd);
+ return -1;
+ }
+ }
+
+ close(fd);
+ return 0;
+ }
+
+ RTE_LOG(ERR, EAL, "Cannot find resource for device\n");
+ close(fd);
+ return -1;
}
-/* map the PCI resource of a PCI device in virtual memory */
+
+/* map the PCI resources of a PCI device in virtual memory */
static int
pci_uio_map_resource(struct rte_pci_device *dev)
{
@@ -278,35 +321,20 @@ pci_uio_map_resource(struct rte_pci_devi
char dirname[PATH_MAX];
char dirname2[PATH_MAX];
char filename[PATH_MAX];
- char devname[PATH_MAX]; /* contains the /dev/uioX */
+ int i, fd;
void *mapaddr;
unsigned uio_num;
- unsigned long size, offset;
+ unsigned long size, offset, page_size;
struct rte_pci_addr *loc = &dev->addr;
- struct uio_resource *uio_res;
+
+ page_size = sysconf(_SC_PAGE_SIZE);
RTE_LOG(DEBUG, EAL, "map PCI resource for device "PCI_PRI_FMT"\n",
loc->domain, loc->bus, loc->devid, loc->function);
/* secondary processes - use already recorded details */
- if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
-
- TAILQ_FOREACH(uio_res, uio_res_list, next) {
- /* skip this element if it doesn't match our PCI address */
- if (memcmp(&uio_res->pci_addr, &dev->addr, sizeof(dev->addr)))
- continue;
-
- if (pci_map_resource(dev, uio_res->addr, uio_res->path, \
- uio_res->offset, uio_res->size) == uio_res->addr)
- return 0;
- else {
- RTE_LOG(ERR, EAL, "Cannot mmap device resource\n");
- return -1;
- }
- }
- RTE_LOG(ERR, EAL, "Cannot find resource for device\n");
- return -1;
- }
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return pci_uio_map_restore(dev);
/* depending on kernel version, uio can be located in uio/uioX
* or uio:uioX */
@@ -362,44 +390,59 @@ pci_uio_map_resource(struct rte_pci_devi
if (e == NULL)
return 0;
- /* get mapping offset */
- rte_snprintf(filename, sizeof(filename),
- "%s/maps/map0/offset", dirname2);
- if (pci_parse_sysfs_value(filename, &offset) < 0) {
- RTE_LOG(ERR, EAL, "%s(): cannot parse offset\n",
- __func__);
- return -1;
- }
+ /* open /dev/uioX */
+ rte_snprintf(dev->uio_name, sizeof(dev->uio_name),
+ "/dev/uio%u", uio_num);
- /* get mapping size */
- rte_snprintf(filename, sizeof(filename),
- "%s/maps/map0/size", dirname2);
- if (pci_parse_sysfs_value(filename, &size) < 0) {
- RTE_LOG(ERR, EAL, "%s(): cannot parse size\n",
- __func__);
+ fd = uio_open(dev->uio_name);
+ if (fd < 0) {
+ RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
+ dev->uio_name, strerror(errno));
return -1;
}
- /* open and mmap /dev/uioX */
- rte_snprintf(devname, sizeof(devname), "/dev/uio%u", uio_num);
- mapaddr = pci_map_resource(dev, NULL, devname, offset, size);
- if (mapaddr == NULL)
- return -1;
- dev->mem_resource.addr = mapaddr;
+ /* map associated memory resources. */
+ for (i = 0; i < PCI_MEM_RESOURCE; i++) {
+ if (dev->mem_resource[i].len == 0)
+ continue;
- /* save the mapping details for secondary processes*/
- uio_res = rte_malloc("UIO_RES", sizeof(*uio_res), 0);
- if (uio_res == NULL){
- RTE_LOG(ERR, EAL, "%s(): cannot store uio mmap details\n", __func__);
- return -1;
+ rte_snprintf(filename, sizeof(filename),
+ "%s/maps/map%d/offset", dirname2, i);
+
+ if (access(filename, F_OK) < 0)
+ continue; /* this resource is not mapped via uio */
+
+ /* get mapping offset */
+ if (pci_parse_sysfs_value(filename, &offset) < 0) {
+ RTE_LOG(ERR, EAL, "%s(): cannot parse offset\n",
+ __func__);
+ return -1;
+ }
+
+ /* page number indicates which resource */
+ offset += i * page_size;
+
+ /* get mapping size */
+ rte_snprintf(filename, sizeof(filename),
+ "%s/maps/map%d/size", dirname2, i);
+ if (pci_parse_sysfs_value(filename, &size) < 0) {
+ RTE_LOG(ERR, EAL, "%s(): cannot parse size\n",
+ __func__);
+ return -1;
+ }
+
+ mapaddr = pci_mmap(fd, NULL, offset, size);
+ if (mapaddr == NULL)
+ return -1;
+
+ dev->mem_resource[i].addr = mapaddr;
+ if (pci_uio_map_save(dev, mapaddr, offset, size) < 0)
+ return -1;
}
- uio_res->addr = mapaddr;
- uio_res->offset = offset;
- uio_res->size = size;
- rte_snprintf(uio_res->path, sizeof(uio_res->path), "%s", devname);
- memcpy(&uio_res->pci_addr, &dev->addr, sizeof(uio_res->pci_addr));
- TAILQ_INSERT_TAIL(uio_res_list, uio_res, next);
+ /* save fd if in primary process */
+ dev->intr_handle.fd = fd;
+ dev->intr_handle.type = RTE_INTR_HANDLE_UIO;
return 0;
}
@@ -420,7 +463,7 @@ pci_parse_sysfs_resource(const char *fil
};
char *ptrs[PCI_RESOURCE_FMT_NVAL];
} res_info;
- int i;
+ int i, m;
uint64_t phys_addr, end_addr, flags;
f = fopen(filename, "r");
@@ -429,6 +472,7 @@ pci_parse_sysfs_resource(const char *fil
return -1;
}
+ m = 0;
for (i = 0; i<PCI_MAX_RESOURCE; i++) {
if (fgets(buf, sizeof(buf), f) == NULL) {
@@ -450,10 +494,16 @@ pci_parse_sysfs_resource(const char *fil
}
if (flags & IORESOURCE_MEM) {
- dev->mem_resource.phys_addr = phys_addr;
- dev->mem_resource.len = end_addr - phys_addr + 1;
- dev->mem_resource.addr = NULL; /* not mapped for now */
- break;
+ if (m == PCI_MEM_RESOURCE) {
+ RTE_LOG(ERR, EAL, "%s(): too many memory resources\n",
+ __func__);
+ goto error;
+ }
+
+ dev->mem_resource[m].phys_addr = phys_addr;
+ dev->mem_resource[m].len = end_addr - phys_addr + 1;
+ dev->mem_resource[m].addr = NULL; /* not mapped for now */
+ ++m;
}
}
fclose(f);
--- a/lib/librte_pmd_ixgbe/ixgbe_ethdev.c 2013-05-29 16:08:07.544000027 -0700
+++ b/lib/librte_pmd_ixgbe/ixgbe_ethdev.c 2013-05-29 16:09:36.355108938 -0700
@@ -369,7 +369,7 @@ eth_ixgbe_dev_init(__attribute__((unused
/* Vendor and Device ID need to be set before init of shared code */
hw->device_id = pci_dev->id.device_id;
hw->vendor_id = pci_dev->id.vendor_id;
- hw->hw_addr = (void *)pci_dev->mem_resource.addr;
+ hw->hw_addr = pci_dev->mem_resource[0].addr;
/* Initialize the shared code */
diag = ixgbe_init_shared_code(hw);
@@ -490,7 +490,7 @@ eth_ixgbevf_dev_init(__attribute__((unus
hw->device_id = pci_dev->id.device_id;
hw->vendor_id = pci_dev->id.vendor_id;
- hw->hw_addr = (void *)pci_dev->mem_resource.addr;
+ hw->hw_addr = pci_dev->mem_resource[0].addr;
/* Initialize the shared code */
diag = ixgbe_init_shared_code(hw);
--- a/app/test-pmd/config.c 2013-03-28 08:50:50.000000000 -0700
+++ b/app/test-pmd/config.c 2013-05-29 16:09:36.359108898 -0700
@@ -180,7 +180,7 @@ port_reg_off_is_invalid(portid_t port_id
(unsigned)reg_off);
return 1;
}
- pci_len = ports[port_id].dev_info.pci_dev->mem_resource.len;
+ pci_len = ports[port_id].dev_info.pci_dev->mem_resource[0].len;
if (reg_off >= pci_len) {
printf("Port %d: register offset %u (0x%X) out of port PCI "
"resource (length=%"PRIu64")\n",
--- a/app/test-pmd/testpmd.h 2013-03-28 08:50:50.000000000 -0700
+++ b/app/test-pmd/testpmd.h 2013-05-29 16:17:33.550417158 -0700
@@ -304,8 +304,8 @@ port_pci_reg_read(struct rte_port *port,
void *reg_addr;
uint32_t reg_v;
- reg_addr = (void *)((char *)port->dev_info.pci_dev->mem_resource.addr +
- reg_off);
+ reg_addr = (char *)port->dev_info.pci_dev->mem_resource[0].addr
+ + reg_off;
reg_v = *((volatile uint32_t *)reg_addr);
return rte_le_to_cpu_32(reg_v);
}
@@ -318,8 +318,8 @@ port_pci_reg_write(struct rte_port *port
{
void *reg_addr;
- reg_addr = (void *)((char *)port->dev_info.pci_dev->mem_resource.addr +
- reg_off);
+ reg_addr = (char *)port->dev_info.pci_dev->mem_resource[0].addr
+ + reg_off;
*((volatile uint32_t *)reg_addr) = rte_cpu_to_le_32(reg_v);
}
next prev parent reply other threads:[~2013-05-30 17:21 UTC|newest]
Thread overview: 39+ messages / expand[flat|nested] mbox.gz Atom feed top
2013-05-30 17:12 [dpdk-dev] [PATCH 0/7] Vyatta patches Stephen Hemminger
2013-05-30 17:12 ` [dpdk-dev] [PATCH 1/7] [PATCH 4/8] igb: workaround errata with wthresh on 82576 Stephen Hemminger
2013-06-05 14:22 ` Vincent JARDIN
2013-06-12 10:06 ` [dpdk-dev] [PATCH 1/7] " Thomas Monjalon
2013-05-30 17:12 ` [dpdk-dev] [PATCH 2/7] rte_timer: optimize for empty case Stephen Hemminger
2013-06-05 14:25 ` Vincent JARDIN
2013-06-12 10:07 ` Thomas Monjalon
2013-05-30 17:12 ` [dpdk-dev] [PATCH 3/7] optimize log/panic Stephen Hemminger
2013-06-05 14:34 ` Vincent JARDIN
2013-06-12 10:09 ` Thomas Monjalon
2013-05-30 17:12 ` [dpdk-dev] [PATCH 4/7] eal: support different modules Stephen Hemminger
2013-06-03 8:58 ` Damien Millescamps
2013-06-03 15:41 ` Stephen Hemminger
2013-06-03 16:36 ` Thomas Monjalon
2013-06-03 17:26 ` Stephen Hemminger
2013-06-04 9:17 ` Damien Millescamps
2013-06-03 16:08 ` Antti Kantee
2013-06-03 16:29 ` Thomas Monjalon
2013-06-03 17:25 ` Stephen Hemminger
2013-06-03 18:40 ` Antti Kantee
2013-05-30 17:12 ` Stephen Hemminger [this message]
2013-06-03 16:41 ` [dpdk-dev] [PATCH 5/7] pci: support multiple PCI regions per device Thomas Monjalon
2013-06-05 14:50 ` Damien Millescamps
2013-06-05 15:49 ` Stephen Hemminger
2013-06-05 18:05 ` Damien Millescamps
2013-06-05 21:33 ` Stephen Hemminger
2013-06-18 1:28 ` somnath kotur
2013-07-16 8:53 ` Thomas Monjalon
2013-07-19 16:44 ` Stephen Hemminger
2013-05-30 17:12 ` [dpdk-dev] [PATCH 6/7] igb_uio: pci_block_user_cfg_access is unsafe, remove it Stephen Hemminger
2013-05-30 17:12 ` [dpdk-dev] [PATCH 7/7] eal: add ability to override DPDK syslog parameters Stephen Hemminger
2013-06-05 14:36 ` Vincent JARDIN
2013-06-12 10:18 ` Thomas Monjalon
2013-05-30 22:20 ` [dpdk-dev] [PATCH 0/7] Vyatta patches Thomas Monjalon
2013-05-31 9:29 ` Damien Millescamps
2013-05-31 15:45 ` Stephen Hemminger
2013-05-31 16:44 ` Damien Millescamps
2013-05-31 17:00 ` Stephen Hemminger
2013-06-03 15:22 ` Thomas Monjalon
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20130530171627.005239011@vyatta.com \
--to=shemminger@vyatta.com \
--cc=dev@dpdk.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).