From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from ns.mahan.org (unknown [67.116.10.138]) by dpdk.org (Postfix) with ESMTP id B4B3E1F3 for ; Fri, 9 Aug 2013 21:46:07 +0200 (CEST) Received: from gypsy.mahan.org (localhost [127.0.0.1]) by ns.mahan.org (8.14.5/8.14.5) with ESMTP id r79JkB1m011699; Fri, 9 Aug 2013 12:46:14 -0700 (PDT) (envelope-from mahan@mahan.org) Message-ID: <52054701.2050602@mahan.org> Date: Fri, 09 Aug 2013 12:46:09 -0700 From: Patrick Mahan User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:17.0) Gecko/20130801 Thunderbird/17.0.8 MIME-Version: 1.0 To: Stephen Hemminger References: <20130808165344.18a543b9@nehalam.linuxnetplumber.net> <7866DDBC-AD11-4BF6-83E7-5FB61DD8281D@mahan.org> <20130808175032.7b343765@nehalam.linuxnetplumber.net> In-Reply-To: <20130808175032.7b343765@nehalam.linuxnetplumber.net> Content-Type: text/plain; charset=ISO-8859-1 Content-Transfer-Encoding: 7bit Cc: "dev@dpdk.org" Subject: Re: [dpdk-dev] Need to support mapping in bar1 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 09 Aug 2013 19:46:08 -0000 Stephen, When did you generate this diff? It failed to apply to 1.3.1r2 so I am hand patching instead. But you might want to update to the latest version. Thanks, Patrick On 8/8/13 5:50 PM, Stephen Hemminger wrote: > Still needs a little cleanup (patch is messy). > > Subject: pci: support multiple PCI regions per device > > Need to change PCI code to support multiple I/O regions on a single device. > Some devices like VMXNET3 have multiple PCI memory regions, and some > have none. > > Signed-off-by: Stephen Hemminger > > --- > app/test-pmd/config.c | 2 > app/test-pmd/testpmd.h | 8 - > lib/librte_eal/common/eal_common_pci.c | 11 + > lib/librte_eal/common/include/rte_pci.h | 5 > lib/librte_eal/linuxapp/eal/eal_pci.c | 244 +++++++++++++++++++------------- > lib/librte_pmd_ixgbe/ixgbe_ethdev.c | 4 > 6 files changed, 167 insertions(+), 107 deletions(-) > > --- a/lib/librte_eal/common/eal_common_pci.c 2013-06-05 14:41:46.000000000 -0700 > +++ b/lib/librte_eal/common/eal_common_pci.c 2013-08-08 17:47:56.541720272 -0700 > @@ -121,12 +121,19 @@ rte_eal_pci_probe(void) > static int > pci_dump_one_device(struct rte_pci_device *dev) > { > + int i; > + > printf(PCI_PRI_FMT, dev->addr.domain, dev->addr.bus, > dev->addr.devid, dev->addr.function); > printf(" - vendor:%x device:%x\n", dev->id.vendor_id, > dev->id.device_id); > - printf(" %16.16"PRIx64" %16.16"PRIx64"\n", > - dev->mem_resource.phys_addr, dev->mem_resource.len); > + > + for (i = 0; i < PCI_MAX_RESOURCE; i++) { > + printf(" %16.16"PRIx64" %16.16"PRIx64"\n", > + dev->mem_resource[i].phys_addr, > + dev->mem_resource[i].len); > + } > + > return 0; > } > > --- a/lib/librte_eal/common/include/rte_pci.h 2013-08-08 17:41:37.866416535 -0700 > +++ b/lib/librte_eal/common/include/rte_pci.h 2013-08-08 17:42:34.585716365 -0700 > @@ -50,6 +50,7 @@ extern "C" { > #include > #include > #include > +#include > #include > > TAILQ_HEAD(pci_device_list, rte_pci_device); /**< PCI devices in D-linked Q. */ > @@ -110,10 +111,12 @@ struct rte_pci_device { > TAILQ_ENTRY(rte_pci_device) next; /**< Next probed PCI device. */ > struct rte_pci_addr addr; /**< PCI location. */ > struct rte_pci_id id; /**< PCI ID. */ > - struct rte_pci_resource mem_resource; /**< PCI Memory Resource */ > + struct rte_pci_resource mem_resource[PCI_MAX_RESOURCE]; > + /**< PCI Memory Resource */ > struct rte_intr_handle intr_handle; /**< Interrupt handle */ > const struct rte_pci_driver *driver; /**< Associated driver */ > unsigned int blacklisted:1; /**< Device is blacklisted */ > + char uio_name[PATH_MAX]; /**< Associated UIO device name */ > }; > > /** Any PCI device identifier (vendor, device, ...) */ > --- a/lib/librte_eal/linuxapp/eal/eal_pci.c 2013-08-08 17:40:52.606974358 -0700 > +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c 2013-08-08 17:47:28.942064203 -0700 > @@ -99,7 +99,6 @@ struct uio_resource { > > struct rte_pci_addr pci_addr; > void *addr; > - char path[PATH_MAX]; > unsigned long size; > unsigned long offset; > }; > @@ -212,64 +211,108 @@ pci_uio_bind_device(struct rte_pci_devic > return 0; > } > > -/* map a particular resource from a file */ > -static void * > -pci_map_resource(struct rte_pci_device *dev, void *requested_addr, const char *devname, > - unsigned long offset, unsigned long size) > +/* > + * open devname: it can take some time to > + * appear, so we wait some time before returning an error > + */ > +static int uio_open(const char *devname) > { > - unsigned n; > - int fd; > - void *mapaddr; > + int n, fd; > > - /* > - * open devname, and mmap it: it can take some time to > - * appear, so we wait some time before returning an error > - */ > - for (n=0; n + for (n=0; n < UIO_DEV_WAIT_TIMEOUT*10; n++) { > fd = open(devname, O_RDWR); > if (fd >= 0) > - break; > + return fd; > + > if (errno != ENOENT) > break; > usleep(100000); > } > - if (fd < 0) { > - RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", devname, strerror(errno)); > - goto fail; > - } > + return -1; > +} > + > +/* map a particular resource from a file */ > +static void * > +pci_mmap(int fd, void *addr, off_t offset, size_t size) > +{ > + void *mapaddr; > > /* Map the PCI memory resource of device */ > - mapaddr = mmap(requested_addr, size, PROT_READ | PROT_WRITE, > - MAP_SHARED, fd, offset); > - if (mapaddr == MAP_FAILED || > - (requested_addr != NULL && mapaddr != requested_addr)) { > - RTE_LOG(ERR, EAL, "%s(): cannot mmap %s: %s\n", __func__, > - devname, strerror(errno)); > - close(fd); > - goto fail; > - } > - if (rte_eal_process_type() == RTE_PROC_PRIMARY) { > - /* save fd if in primary process */ > - dev->intr_handle.fd = fd; > - dev->intr_handle.type = RTE_INTR_HANDLE_UIO; > - } else { > - /* fd is not needed in slave process, close it */ > - dev->intr_handle.fd = -1; > - dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; > - close(fd); > + mapaddr = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_SHARED, > + fd, offset); > + if (mapaddr == MAP_FAILED || (addr != NULL && mapaddr != addr)) { > + RTE_LOG(ERR, EAL, "%s(): cannot mmap %zd@0x%lx: %s\n", > + __func__, size, offset, strerror(errno)); > + return NULL; > } > > RTE_LOG(DEBUG, EAL, "PCI memory mapped at %p\n", mapaddr); > - > return mapaddr; > +} > + > +/* save the mapping details for secondary processes*/ > +static int pci_uio_map_save(const struct rte_pci_device *dev, void *mapaddr, > + unsigned long offset, unsigned long size) > +{ > + struct uio_resource *uio_res; > + > + uio_res = rte_malloc("UIO_RES", sizeof(*uio_res), 0); > + if (uio_res == NULL) { > + RTE_LOG(ERR, EAL, "%s(): cannot store uio mmap details\n", > + __func__); > + return -1; > + } > + > + uio_res->addr = mapaddr; > + uio_res->offset = offset; > + uio_res->size = size; > + memcpy(&uio_res->pci_addr, &dev->addr, sizeof(uio_res->pci_addr)); > + > + TAILQ_INSERT_TAIL(uio_res_list, uio_res, next); > + return 0; > +} > + > +static int pci_uio_map_restore(struct rte_pci_device *dev) > +{ > + struct uio_resource *uio_res; > + int i, fd; > + void *addr; > > -fail: > dev->intr_handle.fd = -1; > dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; > > - return NULL; > + fd = uio_open(dev->uio_name); > + if (fd < 0) > + return -1; > + > + TAILQ_FOREACH(uio_res, uio_res_list, next) { > + /* skip this element if it doesn't match our PCI address */ > + if (memcmp(&uio_res->pci_addr, &dev->addr, sizeof(dev->addr))) > + continue; > + > + for (i = 0; i < PCI_MAX_RESOURCE; i++) { > + if (dev->mem_resource[i].len == 0) > + continue; > + > + addr = pci_mmap(fd, uio_res->addr, > + uio_res->offset, uio_res->size); > + if (addr != uio_res->addr) { > + RTE_LOG(ERR, EAL, "Cannot mmap device resource\n"); > + close(fd); > + return -1; > + } > + } > + > + close(fd); > + return 0; > + } > + > + RTE_LOG(ERR, EAL, "Cannot find resource for device\n"); > + close(fd); > + return -1; > } > -/* map the PCI resource of a PCI device in virtual memory */ > + > +/* map the PCI resources of a PCI device in virtual memory */ > static int > pci_uio_map_resource(struct rte_pci_device *dev) > { > @@ -278,35 +321,20 @@ pci_uio_map_resource(struct rte_pci_devi > char dirname[PATH_MAX]; > char dirname2[PATH_MAX]; > char filename[PATH_MAX]; > - char devname[PATH_MAX]; /* contains the /dev/uioX */ > + int i, fd; > void *mapaddr; > unsigned uio_num; > - unsigned long size, offset; > + unsigned long size, offset, page_size; > struct rte_pci_addr *loc = &dev->addr; > - struct uio_resource *uio_res; > + > + page_size = sysconf(_SC_PAGE_SIZE); > > RTE_LOG(DEBUG, EAL, "map PCI resource for device "PCI_PRI_FMT"\n", > loc->domain, loc->bus, loc->devid, loc->function); > > /* secondary processes - use already recorded details */ > - if (rte_eal_process_type() != RTE_PROC_PRIMARY) { > - > - TAILQ_FOREACH(uio_res, uio_res_list, next) { > - /* skip this element if it doesn't match our PCI address */ > - if (memcmp(&uio_res->pci_addr, &dev->addr, sizeof(dev->addr))) > - continue; > - > - if (pci_map_resource(dev, uio_res->addr, uio_res->path, \ > - uio_res->offset, uio_res->size) == uio_res->addr) > - return 0; > - else { > - RTE_LOG(ERR, EAL, "Cannot mmap device resource\n"); > - return -1; > - } > - } > - RTE_LOG(ERR, EAL, "Cannot find resource for device\n"); > - return -1; > - } > + if (rte_eal_process_type() != RTE_PROC_PRIMARY) > + return pci_uio_map_restore(dev); > > /* depending on kernel version, uio can be located in uio/uioX > * or uio:uioX */ > @@ -362,44 +390,59 @@ pci_uio_map_resource(struct rte_pci_devi > if (e == NULL) > return 0; > > - /* get mapping offset */ > - rte_snprintf(filename, sizeof(filename), > - "%s/maps/map0/offset", dirname2); > - if (pci_parse_sysfs_value(filename, &offset) < 0) { > - RTE_LOG(ERR, EAL, "%s(): cannot parse offset\n", > - __func__); > - return -1; > - } > + /* open /dev/uioX */ > + rte_snprintf(dev->uio_name, sizeof(dev->uio_name), > + "/dev/uio%u", uio_num); > > - /* get mapping size */ > - rte_snprintf(filename, sizeof(filename), > - "%s/maps/map0/size", dirname2); > - if (pci_parse_sysfs_value(filename, &size) < 0) { > - RTE_LOG(ERR, EAL, "%s(): cannot parse size\n", > - __func__); > + fd = uio_open(dev->uio_name); > + if (fd < 0) { > + RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", > + dev->uio_name, strerror(errno)); > return -1; > } > > - /* open and mmap /dev/uioX */ > - rte_snprintf(devname, sizeof(devname), "/dev/uio%u", uio_num); > - mapaddr = pci_map_resource(dev, NULL, devname, offset, size); > - if (mapaddr == NULL) > - return -1; > - dev->mem_resource.addr = mapaddr; > + /* map associated memory resources. */ > + for (i = 0; i < PCI_MAX_RESOURCE; i++) { > + if (dev->mem_resource[i].len == 0) > + continue; > > - /* save the mapping details for secondary processes*/ > - uio_res = rte_malloc("UIO_RES", sizeof(*uio_res), 0); > - if (uio_res == NULL){ > - RTE_LOG(ERR, EAL, "%s(): cannot store uio mmap details\n", __func__); > - return -1; > + rte_snprintf(filename, sizeof(filename), > + "%s/maps/map%d/offset", dirname2, i); > + > + if (access(filename, F_OK) < 0) > + continue; /* this resource is not mapped via uio */ > + > + /* get mapping offset */ > + if (pci_parse_sysfs_value(filename, &offset) < 0) { > + RTE_LOG(ERR, EAL, "%s(): cannot parse offset\n", > + __func__); > + return -1; > + } > + > + /* page number indicates which resource */ > + offset += i * page_size; > + > + /* get mapping size */ > + rte_snprintf(filename, sizeof(filename), > + "%s/maps/map%d/size", dirname2, i); > + if (pci_parse_sysfs_value(filename, &size) < 0) { > + RTE_LOG(ERR, EAL, "%s(): cannot parse size\n", > + __func__); > + return -1; > + } > + > + mapaddr = pci_mmap(fd, NULL, offset, size); > + if (mapaddr == NULL) > + return -1; > + > + dev->mem_resource[i].addr = mapaddr; > + if (pci_uio_map_save(dev, mapaddr, offset, size) < 0) > + return -1; > } > - uio_res->addr = mapaddr; > - uio_res->offset = offset; > - uio_res->size = size; > - rte_snprintf(uio_res->path, sizeof(uio_res->path), "%s", devname); > - memcpy(&uio_res->pci_addr, &dev->addr, sizeof(uio_res->pci_addr)); > > - TAILQ_INSERT_TAIL(uio_res_list, uio_res, next); > + /* save fd if in primary process */ > + dev->intr_handle.fd = fd; > + dev->intr_handle.type = RTE_INTR_HANDLE_UIO; > > return 0; > } > @@ -420,7 +463,7 @@ pci_parse_sysfs_resource(const char *fil > }; > char *ptrs[PCI_RESOURCE_FMT_NVAL]; > } res_info; > - int i; > + int i, m; > uint64_t phys_addr, end_addr, flags; > > f = fopen(filename, "r"); > @@ -429,6 +472,7 @@ pci_parse_sysfs_resource(const char *fil > return -1; > } > > + m = 0; > for (i = 0; i > if (fgets(buf, sizeof(buf), f) == NULL) { > @@ -450,10 +494,16 @@ pci_parse_sysfs_resource(const char *fil > } > > if (flags & IORESOURCE_MEM) { > - dev->mem_resource.phys_addr = phys_addr; > - dev->mem_resource.len = end_addr - phys_addr + 1; > - dev->mem_resource.addr = NULL; /* not mapped for now */ > - break; > + if (m == PCI_MAX_RESOURCE) { > + RTE_LOG(ERR, EAL, "%s(): too many memory resources\n", > + __func__); > + goto error; > + } > + > + dev->mem_resource[m].phys_addr = phys_addr; > + dev->mem_resource[m].len = end_addr - phys_addr + 1; > + dev->mem_resource[m].addr = NULL; /* not mapped for now */ > + ++m; > } > } > fclose(f); > --- a/lib/librte_pmd_ixgbe/ixgbe_ethdev.c 2013-08-08 17:40:52.606974358 -0700 > +++ b/lib/librte_pmd_ixgbe/ixgbe_ethdev.c 2013-08-08 17:41:50.266263568 -0700 > @@ -369,7 +369,7 @@ eth_ixgbe_dev_init(__attribute__((unused > /* Vendor and Device ID need to be set before init of shared code */ > hw->device_id = pci_dev->id.device_id; > hw->vendor_id = pci_dev->id.vendor_id; > - hw->hw_addr = (void *)pci_dev->mem_resource.addr; > + hw->hw_addr = pci_dev->mem_resource[0].addr; > > /* Initialize the shared code */ > diag = ixgbe_init_shared_code(hw); > @@ -490,7 +490,7 @@ eth_ixgbevf_dev_init(__attribute__((unus > > hw->device_id = pci_dev->id.device_id; > hw->vendor_id = pci_dev->id.vendor_id; > - hw->hw_addr = (void *)pci_dev->mem_resource.addr; > + hw->hw_addr = pci_dev->mem_resource[0].addr; > > /* Initialize the shared code */ > diag = ixgbe_init_shared_code(hw); > --- a/app/test-pmd/config.c 2013-06-05 14:41:46.000000000 -0700 > +++ b/app/test-pmd/config.c 2013-08-08 17:41:50.294263224 -0700 > @@ -180,7 +180,7 @@ port_reg_off_is_invalid(portid_t port_id > (unsigned)reg_off); > return 1; > } > - pci_len = ports[port_id].dev_info.pci_dev->mem_resource.len; > + pci_len = ports[port_id].dev_info.pci_dev->mem_resource[0].len; > if (reg_off >= pci_len) { > printf("Port %d: register offset %u (0x%X) out of port PCI " > "resource (length=%"PRIu64")\n", > --- a/app/test-pmd/testpmd.h 2013-06-05 14:41:46.000000000 -0700 > +++ b/app/test-pmd/testpmd.h 2013-08-08 17:41:50.294263224 -0700 > @@ -304,8 +304,8 @@ port_pci_reg_read(struct rte_port *port, > void *reg_addr; > uint32_t reg_v; > > - reg_addr = (void *)((char *)port->dev_info.pci_dev->mem_resource.addr + > - reg_off); > + reg_addr = (char *)port->dev_info.pci_dev->mem_resource[0].addr > + + reg_off; > reg_v = *((volatile uint32_t *)reg_addr); > return rte_le_to_cpu_32(reg_v); > } > @@ -318,8 +318,8 @@ port_pci_reg_write(struct rte_port *port > { > void *reg_addr; > > - reg_addr = (void *)((char *)port->dev_info.pci_dev->mem_resource.addr + > - reg_off); > + reg_addr = (char *)port->dev_info.pci_dev->mem_resource[0].addr > + + reg_off; > *((volatile uint32_t *)reg_addr) = rte_cpu_to_le_32(reg_v); > } > >