From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-pb0-f42.google.com (mail-pb0-f42.google.com [209.85.160.42]) by dpdk.org (Postfix) with ESMTP id D15A51F3 for ; Fri, 9 Aug 2013 02:50:09 +0200 (CEST) Received: by mail-pb0-f42.google.com with SMTP id un15so4097160pbc.29 for ; Thu, 08 Aug 2013 17:50:36 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20120113; h=x-gm-message-state:date:from:to:cc:subject:message-id:in-reply-to :references:mime-version:content-type:content-transfer-encoding; bh=lB5re00S/z4PTHaKQCP9IF0pE7C3F1nEe2ASfdFluOk=; b=kNvWUsDX2ftkH8Mm5IRFaH2jr9MX4rJbPDXNwwYz5VuVVk9ujiMsZh1zf5rvy5CqMi M+/rO5FPlZktkmcBVIstHzBSOnqNG6CoQQvHdxQSLRkRvWTRK2SqR0/tcaNJif3pBpaK Aioztd8EDriEFeYfVv5QenJB5siHifOu5YP0wZ9LOeWQxMk4hTRjdinoBLDPWtU7Ri7B cL7gX8sU2Gih/iTlkwyYl4P1ZtUO4h1U21MNPveIrEy8f99MrsfY+gnCTzz2nksNTcTw ducXRaa0cAw3J9cY87GDy0uoiV9jHC0xRn2e9J4Jrj0VR8UQkuGUETIHkELEL56uiqsf WWnQ== X-Gm-Message-State: ALoCoQlS56AndgSWVPIh0VPGWLN3YIlLvZxLWFyminjt7Rhn+I4qAQCed+bQz2BdIcLwc4uZXXDh X-Received: by 10.67.21.229 with SMTP id hn5mr8768746pad.135.1376009436373; Thu, 08 Aug 2013 17:50:36 -0700 (PDT) Received: from nehalam.linuxnetplumber.net (static-50-53-69-237.bvtn.or.frontiernet.net. [50.53.69.237]) by mx.google.com with ESMTPSA id e7sm16816291pbc.11.2013.08.08.17.50.35 for (version=TLSv1.2 cipher=RC4-SHA bits=128/128); Thu, 08 Aug 2013 17:50:35 -0700 (PDT) Date: Thu, 8 Aug 2013 17:50:32 -0700 From: Stephen Hemminger To: Patrick Mahan Message-ID: <20130808175032.7b343765@nehalam.linuxnetplumber.net> In-Reply-To: <7866DDBC-AD11-4BF6-83E7-5FB61DD8281D@mahan.org> References: <20130808165344.18a543b9@nehalam.linuxnetplumber.net> <7866DDBC-AD11-4BF6-83E7-5FB61DD8281D@mahan.org> X-Mailer: Claws Mail 3.8.1 (GTK+ 2.24.10; x86_64-pc-linux-gnu) Mime-Version: 1.0 Content-Type: text/plain; charset=US-ASCII Content-Transfer-Encoding: 7bit Cc: "dev@dpdk.org" Subject: Re: [dpdk-dev] Need to support mapping in bar1 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 09 Aug 2013 00:50:10 -0000 Still needs a little cleanup (patch is messy). Subject: pci: support multiple PCI regions per device Need to change PCI code to support multiple I/O regions on a single device. Some devices like VMXNET3 have multiple PCI memory regions, and some have none. Signed-off-by: Stephen Hemminger --- app/test-pmd/config.c | 2 app/test-pmd/testpmd.h | 8 - lib/librte_eal/common/eal_common_pci.c | 11 + lib/librte_eal/common/include/rte_pci.h | 5 lib/librte_eal/linuxapp/eal/eal_pci.c | 244 +++++++++++++++++++------------- lib/librte_pmd_ixgbe/ixgbe_ethdev.c | 4 6 files changed, 167 insertions(+), 107 deletions(-) --- a/lib/librte_eal/common/eal_common_pci.c 2013-06-05 14:41:46.000000000 -0700 +++ b/lib/librte_eal/common/eal_common_pci.c 2013-08-08 17:47:56.541720272 -0700 @@ -121,12 +121,19 @@ rte_eal_pci_probe(void) static int pci_dump_one_device(struct rte_pci_device *dev) { + int i; + printf(PCI_PRI_FMT, dev->addr.domain, dev->addr.bus, dev->addr.devid, dev->addr.function); printf(" - vendor:%x device:%x\n", dev->id.vendor_id, dev->id.device_id); - printf(" %16.16"PRIx64" %16.16"PRIx64"\n", - dev->mem_resource.phys_addr, dev->mem_resource.len); + + for (i = 0; i < PCI_MAX_RESOURCE; i++) { + printf(" %16.16"PRIx64" %16.16"PRIx64"\n", + dev->mem_resource[i].phys_addr, + dev->mem_resource[i].len); + } + return 0; } --- a/lib/librte_eal/common/include/rte_pci.h 2013-08-08 17:41:37.866416535 -0700 +++ b/lib/librte_eal/common/include/rte_pci.h 2013-08-08 17:42:34.585716365 -0700 @@ -50,6 +50,7 @@ extern "C" { #include #include #include +#include #include TAILQ_HEAD(pci_device_list, rte_pci_device); /**< PCI devices in D-linked Q. */ @@ -110,10 +111,12 @@ struct rte_pci_device { TAILQ_ENTRY(rte_pci_device) next; /**< Next probed PCI device. */ struct rte_pci_addr addr; /**< PCI location. */ struct rte_pci_id id; /**< PCI ID. */ - struct rte_pci_resource mem_resource; /**< PCI Memory Resource */ + struct rte_pci_resource mem_resource[PCI_MAX_RESOURCE]; + /**< PCI Memory Resource */ struct rte_intr_handle intr_handle; /**< Interrupt handle */ const struct rte_pci_driver *driver; /**< Associated driver */ unsigned int blacklisted:1; /**< Device is blacklisted */ + char uio_name[PATH_MAX]; /**< Associated UIO device name */ }; /** Any PCI device identifier (vendor, device, ...) */ --- a/lib/librte_eal/linuxapp/eal/eal_pci.c 2013-08-08 17:40:52.606974358 -0700 +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c 2013-08-08 17:47:28.942064203 -0700 @@ -99,7 +99,6 @@ struct uio_resource { struct rte_pci_addr pci_addr; void *addr; - char path[PATH_MAX]; unsigned long size; unsigned long offset; }; @@ -212,64 +211,108 @@ pci_uio_bind_device(struct rte_pci_devic return 0; } -/* map a particular resource from a file */ -static void * -pci_map_resource(struct rte_pci_device *dev, void *requested_addr, const char *devname, - unsigned long offset, unsigned long size) +/* + * open devname: it can take some time to + * appear, so we wait some time before returning an error + */ +static int uio_open(const char *devname) { - unsigned n; - int fd; - void *mapaddr; + int n, fd; - /* - * open devname, and mmap it: it can take some time to - * appear, so we wait some time before returning an error - */ - for (n=0; n= 0) - break; + return fd; + if (errno != ENOENT) break; usleep(100000); } - if (fd < 0) { - RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", devname, strerror(errno)); - goto fail; - } + return -1; +} + +/* map a particular resource from a file */ +static void * +pci_mmap(int fd, void *addr, off_t offset, size_t size) +{ + void *mapaddr; /* Map the PCI memory resource of device */ - mapaddr = mmap(requested_addr, size, PROT_READ | PROT_WRITE, - MAP_SHARED, fd, offset); - if (mapaddr == MAP_FAILED || - (requested_addr != NULL && mapaddr != requested_addr)) { - RTE_LOG(ERR, EAL, "%s(): cannot mmap %s: %s\n", __func__, - devname, strerror(errno)); - close(fd); - goto fail; - } - if (rte_eal_process_type() == RTE_PROC_PRIMARY) { - /* save fd if in primary process */ - dev->intr_handle.fd = fd; - dev->intr_handle.type = RTE_INTR_HANDLE_UIO; - } else { - /* fd is not needed in slave process, close it */ - dev->intr_handle.fd = -1; - dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; - close(fd); + mapaddr = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_SHARED, + fd, offset); + if (mapaddr == MAP_FAILED || (addr != NULL && mapaddr != addr)) { + RTE_LOG(ERR, EAL, "%s(): cannot mmap %zd@0x%lx: %s\n", + __func__, size, offset, strerror(errno)); + return NULL; } RTE_LOG(DEBUG, EAL, "PCI memory mapped at %p\n", mapaddr); - return mapaddr; +} + +/* save the mapping details for secondary processes*/ +static int pci_uio_map_save(const struct rte_pci_device *dev, void *mapaddr, + unsigned long offset, unsigned long size) +{ + struct uio_resource *uio_res; + + uio_res = rte_malloc("UIO_RES", sizeof(*uio_res), 0); + if (uio_res == NULL) { + RTE_LOG(ERR, EAL, "%s(): cannot store uio mmap details\n", + __func__); + return -1; + } + + uio_res->addr = mapaddr; + uio_res->offset = offset; + uio_res->size = size; + memcpy(&uio_res->pci_addr, &dev->addr, sizeof(uio_res->pci_addr)); + + TAILQ_INSERT_TAIL(uio_res_list, uio_res, next); + return 0; +} + +static int pci_uio_map_restore(struct rte_pci_device *dev) +{ + struct uio_resource *uio_res; + int i, fd; + void *addr; -fail: dev->intr_handle.fd = -1; dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; - return NULL; + fd = uio_open(dev->uio_name); + if (fd < 0) + return -1; + + TAILQ_FOREACH(uio_res, uio_res_list, next) { + /* skip this element if it doesn't match our PCI address */ + if (memcmp(&uio_res->pci_addr, &dev->addr, sizeof(dev->addr))) + continue; + + for (i = 0; i < PCI_MAX_RESOURCE; i++) { + if (dev->mem_resource[i].len == 0) + continue; + + addr = pci_mmap(fd, uio_res->addr, + uio_res->offset, uio_res->size); + if (addr != uio_res->addr) { + RTE_LOG(ERR, EAL, "Cannot mmap device resource\n"); + close(fd); + return -1; + } + } + + close(fd); + return 0; + } + + RTE_LOG(ERR, EAL, "Cannot find resource for device\n"); + close(fd); + return -1; } -/* map the PCI resource of a PCI device in virtual memory */ + +/* map the PCI resources of a PCI device in virtual memory */ static int pci_uio_map_resource(struct rte_pci_device *dev) { @@ -278,35 +321,20 @@ pci_uio_map_resource(struct rte_pci_devi char dirname[PATH_MAX]; char dirname2[PATH_MAX]; char filename[PATH_MAX]; - char devname[PATH_MAX]; /* contains the /dev/uioX */ + int i, fd; void *mapaddr; unsigned uio_num; - unsigned long size, offset; + unsigned long size, offset, page_size; struct rte_pci_addr *loc = &dev->addr; - struct uio_resource *uio_res; + + page_size = sysconf(_SC_PAGE_SIZE); RTE_LOG(DEBUG, EAL, "map PCI resource for device "PCI_PRI_FMT"\n", loc->domain, loc->bus, loc->devid, loc->function); /* secondary processes - use already recorded details */ - if (rte_eal_process_type() != RTE_PROC_PRIMARY) { - - TAILQ_FOREACH(uio_res, uio_res_list, next) { - /* skip this element if it doesn't match our PCI address */ - if (memcmp(&uio_res->pci_addr, &dev->addr, sizeof(dev->addr))) - continue; - - if (pci_map_resource(dev, uio_res->addr, uio_res->path, \ - uio_res->offset, uio_res->size) == uio_res->addr) - return 0; - else { - RTE_LOG(ERR, EAL, "Cannot mmap device resource\n"); - return -1; - } - } - RTE_LOG(ERR, EAL, "Cannot find resource for device\n"); - return -1; - } + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return pci_uio_map_restore(dev); /* depending on kernel version, uio can be located in uio/uioX * or uio:uioX */ @@ -362,44 +390,59 @@ pci_uio_map_resource(struct rte_pci_devi if (e == NULL) return 0; - /* get mapping offset */ - rte_snprintf(filename, sizeof(filename), - "%s/maps/map0/offset", dirname2); - if (pci_parse_sysfs_value(filename, &offset) < 0) { - RTE_LOG(ERR, EAL, "%s(): cannot parse offset\n", - __func__); - return -1; - } + /* open /dev/uioX */ + rte_snprintf(dev->uio_name, sizeof(dev->uio_name), + "/dev/uio%u", uio_num); - /* get mapping size */ - rte_snprintf(filename, sizeof(filename), - "%s/maps/map0/size", dirname2); - if (pci_parse_sysfs_value(filename, &size) < 0) { - RTE_LOG(ERR, EAL, "%s(): cannot parse size\n", - __func__); + fd = uio_open(dev->uio_name); + if (fd < 0) { + RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", + dev->uio_name, strerror(errno)); return -1; } - /* open and mmap /dev/uioX */ - rte_snprintf(devname, sizeof(devname), "/dev/uio%u", uio_num); - mapaddr = pci_map_resource(dev, NULL, devname, offset, size); - if (mapaddr == NULL) - return -1; - dev->mem_resource.addr = mapaddr; + /* map associated memory resources. */ + for (i = 0; i < PCI_MAX_RESOURCE; i++) { + if (dev->mem_resource[i].len == 0) + continue; - /* save the mapping details for secondary processes*/ - uio_res = rte_malloc("UIO_RES", sizeof(*uio_res), 0); - if (uio_res == NULL){ - RTE_LOG(ERR, EAL, "%s(): cannot store uio mmap details\n", __func__); - return -1; + rte_snprintf(filename, sizeof(filename), + "%s/maps/map%d/offset", dirname2, i); + + if (access(filename, F_OK) < 0) + continue; /* this resource is not mapped via uio */ + + /* get mapping offset */ + if (pci_parse_sysfs_value(filename, &offset) < 0) { + RTE_LOG(ERR, EAL, "%s(): cannot parse offset\n", + __func__); + return -1; + } + + /* page number indicates which resource */ + offset += i * page_size; + + /* get mapping size */ + rte_snprintf(filename, sizeof(filename), + "%s/maps/map%d/size", dirname2, i); + if (pci_parse_sysfs_value(filename, &size) < 0) { + RTE_LOG(ERR, EAL, "%s(): cannot parse size\n", + __func__); + return -1; + } + + mapaddr = pci_mmap(fd, NULL, offset, size); + if (mapaddr == NULL) + return -1; + + dev->mem_resource[i].addr = mapaddr; + if (pci_uio_map_save(dev, mapaddr, offset, size) < 0) + return -1; } - uio_res->addr = mapaddr; - uio_res->offset = offset; - uio_res->size = size; - rte_snprintf(uio_res->path, sizeof(uio_res->path), "%s", devname); - memcpy(&uio_res->pci_addr, &dev->addr, sizeof(uio_res->pci_addr)); - TAILQ_INSERT_TAIL(uio_res_list, uio_res, next); + /* save fd if in primary process */ + dev->intr_handle.fd = fd; + dev->intr_handle.type = RTE_INTR_HANDLE_UIO; return 0; } @@ -420,7 +463,7 @@ pci_parse_sysfs_resource(const char *fil }; char *ptrs[PCI_RESOURCE_FMT_NVAL]; } res_info; - int i; + int i, m; uint64_t phys_addr, end_addr, flags; f = fopen(filename, "r"); @@ -429,6 +472,7 @@ pci_parse_sysfs_resource(const char *fil return -1; } + m = 0; for (i = 0; imem_resource.phys_addr = phys_addr; - dev->mem_resource.len = end_addr - phys_addr + 1; - dev->mem_resource.addr = NULL; /* not mapped for now */ - break; + if (m == PCI_MAX_RESOURCE) { + RTE_LOG(ERR, EAL, "%s(): too many memory resources\n", + __func__); + goto error; + } + + dev->mem_resource[m].phys_addr = phys_addr; + dev->mem_resource[m].len = end_addr - phys_addr + 1; + dev->mem_resource[m].addr = NULL; /* not mapped for now */ + ++m; } } fclose(f); --- a/lib/librte_pmd_ixgbe/ixgbe_ethdev.c 2013-08-08 17:40:52.606974358 -0700 +++ b/lib/librte_pmd_ixgbe/ixgbe_ethdev.c 2013-08-08 17:41:50.266263568 -0700 @@ -369,7 +369,7 @@ eth_ixgbe_dev_init(__attribute__((unused /* Vendor and Device ID need to be set before init of shared code */ hw->device_id = pci_dev->id.device_id; hw->vendor_id = pci_dev->id.vendor_id; - hw->hw_addr = (void *)pci_dev->mem_resource.addr; + hw->hw_addr = pci_dev->mem_resource[0].addr; /* Initialize the shared code */ diag = ixgbe_init_shared_code(hw); @@ -490,7 +490,7 @@ eth_ixgbevf_dev_init(__attribute__((unus hw->device_id = pci_dev->id.device_id; hw->vendor_id = pci_dev->id.vendor_id; - hw->hw_addr = (void *)pci_dev->mem_resource.addr; + hw->hw_addr = pci_dev->mem_resource[0].addr; /* Initialize the shared code */ diag = ixgbe_init_shared_code(hw); --- a/app/test-pmd/config.c 2013-06-05 14:41:46.000000000 -0700 +++ b/app/test-pmd/config.c 2013-08-08 17:41:50.294263224 -0700 @@ -180,7 +180,7 @@ port_reg_off_is_invalid(portid_t port_id (unsigned)reg_off); return 1; } - pci_len = ports[port_id].dev_info.pci_dev->mem_resource.len; + pci_len = ports[port_id].dev_info.pci_dev->mem_resource[0].len; if (reg_off >= pci_len) { printf("Port %d: register offset %u (0x%X) out of port PCI " "resource (length=%"PRIu64")\n", --- a/app/test-pmd/testpmd.h 2013-06-05 14:41:46.000000000 -0700 +++ b/app/test-pmd/testpmd.h 2013-08-08 17:41:50.294263224 -0700 @@ -304,8 +304,8 @@ port_pci_reg_read(struct rte_port *port, void *reg_addr; uint32_t reg_v; - reg_addr = (void *)((char *)port->dev_info.pci_dev->mem_resource.addr + - reg_off); + reg_addr = (char *)port->dev_info.pci_dev->mem_resource[0].addr + + reg_off; reg_v = *((volatile uint32_t *)reg_addr); return rte_le_to_cpu_32(reg_v); } @@ -318,8 +318,8 @@ port_pci_reg_write(struct rte_port *port { void *reg_addr; - reg_addr = (void *)((char *)port->dev_info.pci_dev->mem_resource.addr + - reg_off); + reg_addr = (char *)port->dev_info.pci_dev->mem_resource[0].addr + + reg_off; *((volatile uint32_t *)reg_addr) = rte_cpu_to_le_32(reg_v); }