This patch series adds support to VMBUS & NetVSC PMDs run on FreeBSD Srikanth Kaka (11): bus/vmbus: stub for FreeBSD support bus/vmbus: scan and get the network device bus/vmbus: handle mapping of device resources bus/vmbus: get device resource values using sysctl bus/vmbus: open subchannels net/netvsc: request HV_UIO to open sub-channels bus/vmbus: map the subchannel resources net/netvsc: moving event monitoring support net/netvsc: moving hotplug retry to OS dir bus/vmbus: add meson suport for FreeBSD OS net/netvsc: add meson support for FreeBSD drivers/bus/vmbus/freebsd/vmbus_bus.c | 296 +++++++++++++++ drivers/bus/vmbus/freebsd/vmbus_uio.c | 499 +++++++++++++++++++++++++ drivers/bus/vmbus/linux/vmbus_uio.c | 8 + drivers/bus/vmbus/meson.build | 6 +- drivers/bus/vmbus/private.h | 1 + drivers/bus/vmbus/rte_bus_vmbus.h | 10 + drivers/bus/vmbus/version.map | 1 + drivers/bus/vmbus/vmbus_channel.c | 5 + drivers/net/netvsc/freebsd/hn_os.c | 22 ++ drivers/net/netvsc/freebsd/meson.build | 6 + drivers/net/netvsc/hn_ethdev.c | 95 +---- drivers/net/netvsc/hn_os.h | 8 + drivers/net/netvsc/linux/hn_os.c | 111 ++++++ drivers/net/netvsc/linux/meson.build | 6 + drivers/net/netvsc/meson.build | 4 + 15 files changed, 989 insertions(+), 89 deletions(-) create mode 100644 drivers/bus/vmbus/freebsd/vmbus_bus.c create mode 100644 drivers/bus/vmbus/freebsd/vmbus_uio.c create mode 100644 drivers/net/netvsc/freebsd/hn_os.c create mode 100644 drivers/net/netvsc/freebsd/meson.build create mode 100644 drivers/net/netvsc/hn_os.h create mode 100644 drivers/net/netvsc/linux/hn_os.c create mode 100644 drivers/net/netvsc/linux/meson.build -- 2.30.2
These files are a copy of their Linux equivalents. They will be ported to FreeBSD. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_bus.c | 376 +++++++++++++++++++++ drivers/bus/vmbus/freebsd/vmbus_uio.c | 453 ++++++++++++++++++++++++++ 2 files changed, 829 insertions(+) create mode 100644 drivers/bus/vmbus/freebsd/vmbus_bus.c create mode 100644 drivers/bus/vmbus/freebsd/vmbus_uio.c diff --git a/drivers/bus/vmbus/freebsd/vmbus_bus.c b/drivers/bus/vmbus/freebsd/vmbus_bus.c new file mode 100644 index 0000000000..3c924eee14 --- /dev/null +++ b/drivers/bus/vmbus/freebsd/vmbus_bus.c @@ -0,0 +1,376 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <string.h> +#include <unistd.h> +#include <dirent.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/stat.h> + +#include <rte_eal.h> +#include <rte_uuid.h> +#include <rte_tailq.h> +#include <rte_log.h> +#include <rte_devargs.h> +#include <rte_memory.h> +#include <rte_malloc.h> +#include <rte_bus_vmbus.h> + +#include "eal_filesystem.h" +#include "private.h" + +/** Pathname of VMBUS devices directory. */ +#define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" + +/* + * GUID associated with network devices + * {f8615163-df3e-46c5-913f-f2d2f965ed0e} + */ +static const rte_uuid_t vmbus_nic_uuid = { + 0xf8, 0x61, 0x51, 0x63, + 0xdf, 0x3e, + 0x46, 0xc5, + 0x91, 0x3f, + 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0xe +}; + +extern struct rte_vmbus_bus rte_vmbus_bus; + +/* Read sysfs file to get UUID */ +static int +parse_sysfs_uuid(const char *filename, rte_uuid_t uu) +{ + char buf[BUFSIZ]; + char *cp, *in = buf; + FILE *f; + + f = fopen(filename, "r"); + if (f == NULL) { + VMBUS_LOG(ERR, "cannot open sysfs value %s: %s", + filename, strerror(errno)); + return -1; + } + + if (fgets(buf, sizeof(buf), f) == NULL) { + VMBUS_LOG(ERR, "cannot read sysfs value %s", + filename); + fclose(f); + return -1; + } + fclose(f); + + cp = strchr(buf, '\n'); + if (cp) + *cp = '\0'; + + /* strip { } notation */ + if (buf[0] == '{') { + in = buf + 1; + cp = strchr(in, '}'); + if (cp) + *cp = '\0'; + } + + if (rte_uuid_parse(in, uu) < 0) { + VMBUS_LOG(ERR, "%s %s not a valid UUID", + filename, buf); + return -1; + } + + return 0; +} + +static int +get_sysfs_string(const char *filename, char *buf, size_t buflen) +{ + char *cp; + FILE *f; + + f = fopen(filename, "r"); + if (f == NULL) { + VMBUS_LOG(ERR, "cannot open sysfs value %s:%s", + filename, strerror(errno)); + return -1; + } + + if (fgets(buf, buflen, f) == NULL) { + VMBUS_LOG(ERR, "cannot read sysfs value %s", + filename); + fclose(f); + return -1; + } + fclose(f); + + /* remove trailing newline */ + cp = memchr(buf, '\n', buflen); + if (cp) + *cp = '\0'; + + return 0; +} + +static int +vmbus_get_uio_dev(const struct rte_vmbus_device *dev, + char *dstbuf, size_t buflen) +{ + char dirname[PATH_MAX]; + unsigned int uio_num; + struct dirent *e; + DIR *dir; + + /* Assume recent kernel where uio is in uio/uioX */ + snprintf(dirname, sizeof(dirname), + SYSFS_VMBUS_DEVICES "/%s/uio", dev->device.name); + + dir = opendir(dirname); + if (dir == NULL) + return -1; /* Not a UIO device */ + + /* take the first file starting with "uio" */ + while ((e = readdir(dir)) != NULL) { + const int prefix_len = 3; + char *endptr; + + if (strncmp(e->d_name, "uio", prefix_len) != 0) + continue; + + /* try uio%d */ + errno = 0; + uio_num = strtoull(e->d_name + prefix_len, &endptr, 10); + if (errno == 0 && endptr != (e->d_name + prefix_len)) { + snprintf(dstbuf, buflen, "%s/uio%u", dirname, uio_num); + break; + } + } + closedir(dir); + + if (e == NULL) + return -1; + + return uio_num; +} + +/* Check map names with kernel names */ +static const char *map_names[VMBUS_MAX_RESOURCE] = { + [HV_TXRX_RING_MAP] = "txrx_rings", + [HV_INT_PAGE_MAP] = "int_page", + [HV_MON_PAGE_MAP] = "monitor_page", + [HV_RECV_BUF_MAP] = "recv:", + [HV_SEND_BUF_MAP] = "send:", +}; + + +/* map the resources of a vmbus device in virtual memory */ +int +rte_vmbus_map_device(struct rte_vmbus_device *dev) +{ + char uioname[PATH_MAX], filename[PATH_MAX]; + char dirname[PATH_MAX], mapname[64]; + int i; + + dev->uio_num = vmbus_get_uio_dev(dev, uioname, sizeof(uioname)); + if (dev->uio_num < 0) { + VMBUS_LOG(DEBUG, "Not managed by UIO driver, skipped"); + return 1; + } + + /* Extract resource value */ + for (i = 0; i < VMBUS_MAX_RESOURCE; i++) { + struct rte_mem_resource *res = &dev->resource[i]; + unsigned long len, gpad = 0; + char *cp; + + snprintf(dirname, sizeof(dirname), + "%s/maps/map%d", uioname, i); + + snprintf(filename, sizeof(filename), + "%s/name", dirname); + + if (get_sysfs_string(filename, mapname, sizeof(mapname)) < 0) { + VMBUS_LOG(ERR, "could not read %s", filename); + return -1; + } + + if (strncmp(map_names[i], mapname, strlen(map_names[i])) != 0) { + VMBUS_LOG(ERR, + "unexpected resource %s (expected %s)", + mapname, map_names[i]); + return -1; + } + + snprintf(filename, sizeof(filename), + "%s/size", dirname); + if (eal_parse_sysfs_value(filename, &len) < 0) { + VMBUS_LOG(ERR, + "could not read %s", filename); + return -1; + } + res->len = len; + + /* both send and receive buffers have gpad in name */ + cp = memchr(mapname, ':', sizeof(mapname)); + if (cp) + gpad = strtoul(cp+1, NULL, 0); + + /* put the GPAD value in physical address */ + res->phys_addr = gpad; + } + + return vmbus_uio_map_resource(dev); +} + +void +rte_vmbus_unmap_device(struct rte_vmbus_device *dev) +{ + vmbus_uio_unmap_resource(dev); +} + +/* Scan one vmbus sysfs entry, and fill the devices list from it. */ +static int +vmbus_scan_one(const char *name) +{ + struct rte_vmbus_device *dev, *dev2; + char filename[PATH_MAX]; + char dirname[PATH_MAX]; + unsigned long tmp; + + dev = calloc(1, sizeof(*dev)); + if (dev == NULL) + return -1; + + dev->device.bus = &rte_vmbus_bus.bus; + dev->device.name = strdup(name); + if (!dev->device.name) + goto error; + + /* sysfs base directory + * /sys/bus/vmbus/devices/7a08391f-f5a0-4ac0-9802-d13fd964f8df + * or on older kernel + * /sys/bus/vmbus/devices/vmbus_1 + */ + snprintf(dirname, sizeof(dirname), "%s/%s", + SYSFS_VMBUS_DEVICES, name); + + /* get device class */ + snprintf(filename, sizeof(filename), "%s/class_id", dirname); + if (parse_sysfs_uuid(filename, dev->class_id) < 0) + goto error; + + /* skip non-network devices */ + if (rte_uuid_compare(dev->class_id, vmbus_nic_uuid) != 0) { + free(dev); + return 0; + } + + /* get device id */ + snprintf(filename, sizeof(filename), "%s/device_id", dirname); + if (parse_sysfs_uuid(filename, dev->device_id) < 0) + goto error; + + /* get relid */ + snprintf(filename, sizeof(filename), "%s/id", dirname); + if (eal_parse_sysfs_value(filename, &tmp) < 0) + goto error; + dev->relid = tmp; + + /* get monitor id */ + snprintf(filename, sizeof(filename), "%s/monitor_id", dirname); + if (eal_parse_sysfs_value(filename, &tmp) < 0) + goto error; + dev->monitor_id = tmp; + + /* get numa node (if present) */ + snprintf(filename, sizeof(filename), "%s/numa_node", + dirname); + + if (access(filename, R_OK) == 0) { + if (eal_parse_sysfs_value(filename, &tmp) < 0) + goto error; + dev->device.numa_node = tmp; + } else { + /* if no NUMA support, set default to 0 */ + dev->device.numa_node = SOCKET_ID_ANY; + } + + dev->device.devargs = vmbus_devargs_lookup(dev); + + /* device is valid, add in list (sorted) */ + VMBUS_LOG(DEBUG, "Adding vmbus device %s", name); + + TAILQ_FOREACH(dev2, &rte_vmbus_bus.device_list, next) { + int ret; + + ret = rte_uuid_compare(dev->device_id, dev2->device_id); + if (ret > 0) + continue; + + if (ret < 0) { + vmbus_insert_device(dev2, dev); + } else { /* already registered */ + VMBUS_LOG(NOTICE, + "%s already registered", name); + free(dev); + } + return 0; + } + + vmbus_add_device(dev); + return 0; +error: + VMBUS_LOG(DEBUG, "failed"); + + free(dev); + return -1; +} + +/* + * Scan the content of the vmbus, and the devices in the devices list + */ +int +rte_vmbus_scan(void) +{ + struct dirent *e; + DIR *dir; + + dir = opendir(SYSFS_VMBUS_DEVICES); + if (dir == NULL) { + if (errno == ENOENT) + return 0; + + VMBUS_LOG(ERR, "opendir %s failed: %s", + SYSFS_VMBUS_DEVICES, strerror(errno)); + return -1; + } + + while ((e = readdir(dir)) != NULL) { + if (e->d_name[0] == '.') + continue; + + if (vmbus_scan_one(e->d_name) < 0) + goto error; + } + closedir(dir); + return 0; + +error: + closedir(dir); + return -1; +} + +void rte_vmbus_irq_mask(struct rte_vmbus_device *device) +{ + vmbus_uio_irq_control(device, 1); +} + +void rte_vmbus_irq_unmask(struct rte_vmbus_device *device) +{ + vmbus_uio_irq_control(device, 0); +} + +int rte_vmbus_irq_read(struct rte_vmbus_device *device) +{ + return vmbus_uio_irq_read(device); +} diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c new file mode 100644 index 0000000000..b52ca5bf1d --- /dev/null +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -0,0 +1,453 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <dirent.h> +#include <inttypes.h> +#include <sys/stat.h> +#include <sys/mman.h> + +#include <rte_log.h> +#include <rte_bus.h> +#include <rte_memory.h> +#include <rte_common.h> +#include <rte_malloc.h> +#include <rte_bus_vmbus.h> +#include <rte_string_fns.h> + +#include "private.h" + +/** Pathname of VMBUS devices directory. */ +#define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" + +static void *vmbus_map_addr; + +/* Control interrupts */ +void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff) +{ + if (write(dev->intr_handle.fd, &onoff, sizeof(onoff)) < 0) { + VMBUS_LOG(ERR, "cannot write to %d:%s", + dev->intr_handle.fd, strerror(errno)); + } +} + +int vmbus_uio_irq_read(struct rte_vmbus_device *dev) +{ + int32_t count; + int cc; + + cc = read(dev->intr_handle.fd, &count, sizeof(count)); + if (cc < (int)sizeof(count)) { + if (cc < 0) { + VMBUS_LOG(ERR, "IRQ read failed %s", + strerror(errno)); + return -errno; + } + VMBUS_LOG(ERR, "can't read IRQ count"); + return -EINVAL; + } + + return count; +} + +void +vmbus_uio_free_resource(struct rte_vmbus_device *dev, + struct mapped_vmbus_resource *uio_res) +{ + rte_free(uio_res); + + if (dev->intr_handle.uio_cfg_fd >= 0) { + close(dev->intr_handle.uio_cfg_fd); + dev->intr_handle.uio_cfg_fd = -1; + } + + if (dev->intr_handle.fd >= 0) { + close(dev->intr_handle.fd); + dev->intr_handle.fd = -1; + dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; + } +} + +int +vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, + struct mapped_vmbus_resource **uio_res) +{ + char devname[PATH_MAX]; /* contains the /dev/uioX */ + + /* save fd if in primary process */ + snprintf(devname, sizeof(devname), "/dev/uio%u", dev->uio_num); + dev->intr_handle.fd = open(devname, O_RDWR); + if (dev->intr_handle.fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + devname, strerror(errno)); + goto error; + } + dev->intr_handle.type = RTE_INTR_HANDLE_UIO_INTX; + + /* allocate the mapping details for secondary processes*/ + *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); + if (*uio_res == NULL) { + VMBUS_LOG(ERR, "cannot store uio mmap details"); + goto error; + } + + strlcpy((*uio_res)->path, devname, PATH_MAX); + rte_uuid_copy((*uio_res)->id, dev->device_id); + + return 0; + +error: + vmbus_uio_free_resource(dev, *uio_res); + return -1; +} + +static int +find_max_end_va(const struct rte_memseg_list *msl, void *arg) +{ + size_t sz = msl->memseg_arr.len * msl->page_sz; + void *end_va = RTE_PTR_ADD(msl->base_va, sz); + void **max_va = arg; + + if (*max_va < end_va) + *max_va = end_va; + return 0; +} + +/* + * TODO: this should be part of memseg api. + * code is duplicated from PCI. + */ +static void * +vmbus_find_max_end_va(void) +{ + void *va = NULL; + + rte_memseg_list_walk(find_max_end_va, &va); + return va; +} + +int +vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int idx, + struct mapped_vmbus_resource *uio_res, + int flags) +{ + size_t size = dev->resource[idx].len; + struct vmbus_map *maps = uio_res->maps; + void *mapaddr; + off_t offset; + int fd; + + /* devname for mmap */ + fd = open(uio_res->path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + uio_res->path, strerror(errno)); + return -1; + } + + /* try mapping somewhere close to the end of hugepages */ + if (vmbus_map_addr == NULL) + vmbus_map_addr = vmbus_find_max_end_va(); + + /* offset is special in uio it indicates which resource */ + offset = idx * rte_mem_page_size(); + + mapaddr = vmbus_map_resource(vmbus_map_addr, fd, offset, size, flags); + close(fd); + + if (mapaddr == MAP_FAILED) + return -1; + + dev->resource[idx].addr = mapaddr; + vmbus_map_addr = RTE_PTR_ADD(mapaddr, size); + + /* Record result of successful mapping for use by secondary */ + maps[idx].addr = mapaddr; + maps[idx].size = size; + + return 0; +} + +static int vmbus_uio_map_primary(struct vmbus_channel *chan, + void **ring_buf, uint32_t *ring_size) +{ + struct mapped_vmbus_resource *uio_res; + + uio_res = vmbus_uio_find_resource(chan->device); + if (!uio_res) { + VMBUS_LOG(ERR, "can not find resources!"); + return -ENOMEM; + } + + if (uio_res->nb_maps < VMBUS_MAX_RESOURCE) { + VMBUS_LOG(ERR, "VMBUS: only %u resources found!", + uio_res->nb_maps); + return -EINVAL; + } + + *ring_size = uio_res->maps[HV_TXRX_RING_MAP].size / 2; + *ring_buf = uio_res->maps[HV_TXRX_RING_MAP].addr; + return 0; +} + +static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan, + void **ring_buf, uint32_t *ring_size) +{ + char ring_path[PATH_MAX]; + size_t file_size; + struct stat sb; + void *mapaddr; + int fd; + + snprintf(ring_path, sizeof(ring_path), + "%s/%s/channels/%u/ring", + SYSFS_VMBUS_DEVICES, dev->device.name, + chan->relid); + + fd = open(ring_path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + ring_path, strerror(errno)); + return -errno; + } + + if (fstat(fd, &sb) < 0) { + VMBUS_LOG(ERR, "Cannot state %s: %s", + ring_path, strerror(errno)); + close(fd); + return -errno; + } + file_size = sb.st_size; + + if (file_size == 0 || (file_size & (rte_mem_page_size() - 1))) { + VMBUS_LOG(ERR, "incorrect size %s: %zu", + ring_path, file_size); + + close(fd); + return -EINVAL; + } + + mapaddr = vmbus_map_resource(vmbus_map_addr, fd, + 0, file_size, 0); + close(fd); + + if (mapaddr == MAP_FAILED) + return -EIO; + + *ring_size = file_size / 2; + *ring_buf = mapaddr; + + vmbus_map_addr = RTE_PTR_ADD(mapaddr, file_size); + return 0; +} + +int +vmbus_uio_map_secondary_subchan(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan) +{ + const struct vmbus_br *br = &chan->txbr; + char ring_path[PATH_MAX]; + void *mapaddr, *ring_buf; + uint32_t ring_size; + int fd; + + snprintf(ring_path, sizeof(ring_path), + "%s/%s/channels/%u/ring", + SYSFS_VMBUS_DEVICES, dev->device.name, + chan->relid); + + ring_buf = br->vbr; + ring_size = br->dsize + sizeof(struct vmbus_bufring); + VMBUS_LOG(INFO, "secondary ring_buf %p size %u", + ring_buf, ring_size); + + fd = open(ring_path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + ring_path, strerror(errno)); + return -errno; + } + + mapaddr = vmbus_map_resource(ring_buf, fd, 0, 2 * ring_size, 0); + close(fd); + + if (mapaddr == ring_buf) + return 0; + + if (mapaddr == MAP_FAILED) + VMBUS_LOG(ERR, + "mmap subchan %u in secondary failed", chan->relid); + else { + VMBUS_LOG(ERR, + "mmap subchan %u in secondary address mismatch", + chan->relid); + vmbus_unmap_resource(mapaddr, 2 * ring_size); + } + return -1; +} + +int vmbus_uio_map_rings(struct vmbus_channel *chan) +{ + const struct rte_vmbus_device *dev = chan->device; + uint32_t ring_size; + void *ring_buf; + int ret; + + /* Primary channel */ + if (chan->subchannel_id == 0) + ret = vmbus_uio_map_primary(chan, &ring_buf, &ring_size); + else + ret = vmbus_uio_map_subchan(dev, chan, &ring_buf, &ring_size); + + if (ret) + return ret; + + vmbus_br_setup(&chan->txbr, ring_buf, ring_size); + vmbus_br_setup(&chan->rxbr, (char *)ring_buf + ring_size, ring_size); + return 0; +} + +static int vmbus_uio_sysfs_read(const char *dir, const char *name, + unsigned long *val, unsigned long max_range) +{ + char path[PATH_MAX]; + FILE *f; + int ret; + + snprintf(path, sizeof(path), "%s/%s", dir, name); + f = fopen(path, "r"); + if (!f) { + VMBUS_LOG(ERR, "can't open %s:%s", + path, strerror(errno)); + return -errno; + } + + if (fscanf(f, "%lu", val) != 1) + ret = -EIO; + else if (*val > max_range) + ret = -ERANGE; + else + ret = 0; + fclose(f); + + return ret; +} + +static bool vmbus_uio_ring_present(const struct rte_vmbus_device *dev, + uint32_t relid) +{ + char ring_path[PATH_MAX]; + + /* Check if kernel has subchannel sysfs files */ + snprintf(ring_path, sizeof(ring_path), + "%s/%s/channels/%u/ring", + SYSFS_VMBUS_DEVICES, dev->device.name, relid); + + return access(ring_path, R_OK|W_OK) == 0; +} + +bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan) +{ + return vmbus_uio_ring_present(dev, chan->relid); +} + +static bool vmbus_isnew_subchannel(struct vmbus_channel *primary, + unsigned long id) +{ + const struct vmbus_channel *c; + + STAILQ_FOREACH(c, &primary->subchannel_list, next) { + if (c->relid == id) + return false; + } + return true; +} + +int vmbus_uio_get_subchan(struct vmbus_channel *primary, + struct vmbus_channel **subchan) +{ + const struct rte_vmbus_device *dev = primary->device; + char chan_path[PATH_MAX], subchan_path[PATH_MAX]; + struct dirent *ent; + DIR *chan_dir; + int err; + + snprintf(chan_path, sizeof(chan_path), + "%s/%s/channels", + SYSFS_VMBUS_DEVICES, dev->device.name); + + chan_dir = opendir(chan_path); + if (!chan_dir) { + VMBUS_LOG(ERR, "cannot open %s: %s", + chan_path, strerror(errno)); + return -errno; + } + + while ((ent = readdir(chan_dir))) { + unsigned long relid, subid, monid; + char *endp; + + if (ent->d_name[0] == '.') + continue; + + errno = 0; + relid = strtoul(ent->d_name, &endp, 0); + if (*endp || errno != 0 || relid > UINT16_MAX) { + VMBUS_LOG(NOTICE, "not a valid channel relid: %s", + ent->d_name); + continue; + } + + if (!vmbus_isnew_subchannel(primary, relid)) { + VMBUS_LOG(DEBUG, "skip already found channel: %lu", + relid); + continue; + } + + if (!vmbus_uio_ring_present(dev, relid)) { + VMBUS_LOG(DEBUG, "ring mmap not found (yet) for: %lu", + relid); + continue; + } + + snprintf(subchan_path, sizeof(subchan_path), "%s/%lu", + chan_path, relid); + err = vmbus_uio_sysfs_read(subchan_path, "subchannel_id", + &subid, UINT16_MAX); + if (err) { + VMBUS_LOG(NOTICE, "no subchannel_id in %s:%s", + subchan_path, strerror(-err)); + goto fail; + } + + if (subid == 0) + continue; /* skip primary channel */ + + err = vmbus_uio_sysfs_read(subchan_path, "monitor_id", + &monid, UINT8_MAX); + if (err) { + VMBUS_LOG(NOTICE, "no monitor_id in %s:%s", + subchan_path, strerror(-err)); + goto fail; + } + + err = vmbus_chan_create(dev, relid, subid, monid, subchan); + if (err) { + VMBUS_LOG(ERR, "subchannel setup failed"); + goto fail; + } + break; + } + closedir(chan_dir); + + return (ent == NULL) ? -ENOENT : 0; +fail: + closedir(chan_dir); + return err; +} -- 2.30.2
Using sysctl, all the devices on the VMBUS are identified by the PMD. On finding the Network device's device id, it is added to VMBUS dev list. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_bus.c | 241 +++++++++++++++----------- 1 file changed, 144 insertions(+), 97 deletions(-) diff --git a/drivers/bus/vmbus/freebsd/vmbus_bus.c b/drivers/bus/vmbus/freebsd/vmbus_bus.c index 3c924eee14..8eb428a154 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_bus.c +++ b/drivers/bus/vmbus/freebsd/vmbus_bus.c @@ -22,8 +22,9 @@ #include "eal_filesystem.h" #include "private.h" -/** Pathname of VMBUS devices directory. */ -#define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" +#include <sys/bus.h> +#include <sys/types.h> +#include <sys/sysctl.h> /* * GUID associated with network devices @@ -39,44 +40,15 @@ static const rte_uuid_t vmbus_nic_uuid = { extern struct rte_vmbus_bus rte_vmbus_bus; -/* Read sysfs file to get UUID */ +/* Parse UUID */ static int parse_sysfs_uuid(const char *filename, rte_uuid_t uu) { - char buf[BUFSIZ]; - char *cp, *in = buf; - FILE *f; - - f = fopen(filename, "r"); - if (f == NULL) { - VMBUS_LOG(ERR, "cannot open sysfs value %s: %s", - filename, strerror(errno)); - return -1; - } - - if (fgets(buf, sizeof(buf), f) == NULL) { - VMBUS_LOG(ERR, "cannot read sysfs value %s", - filename); - fclose(f); - return -1; - } - fclose(f); - - cp = strchr(buf, '\n'); - if (cp) - *cp = '\0'; - - /* strip { } notation */ - if (buf[0] == '{') { - in = buf + 1; - cp = strchr(in, '}'); - if (cp) - *cp = '\0'; - } + char in[BUFSIZ]; + strncpy(in, filename, BUFSIZ); if (rte_uuid_parse(in, uu) < 0) { - VMBUS_LOG(ERR, "%s %s not a valid UUID", - filename, buf); + VMBUS_LOG(ERR, "%s not a valid UUID", in); return -1; } @@ -228,35 +200,33 @@ rte_vmbus_unmap_device(struct rte_vmbus_device *dev) vmbus_uio_unmap_resource(dev); } -/* Scan one vmbus sysfs entry, and fill the devices list from it. */ +/* Scan one vmbus entry, and fill the devices list from it. */ static int -vmbus_scan_one(const char *name) +vmbus_scan_one(const char *name, unsigned int unit_num) { struct rte_vmbus_device *dev, *dev2; - char filename[PATH_MAX]; - char dirname[PATH_MAX]; - unsigned long tmp; + char sysctlBuffer[PATH_MAX], sysctlVar[PATH_MAX]; + size_t guid_len = 36, len = PATH_MAX; + char classid[guid_len], deviceid[guid_len]; dev = calloc(1, sizeof(*dev)); if (dev == NULL) return -1; - dev->device.bus = &rte_vmbus_bus.bus; - dev->device.name = strdup(name); - if (!dev->device.name) + /* get class id and device id */ + snprintf(sysctlVar, len, "dev.%s.%u.%%pnpinfo", name, unit_num); + if (sysctlbyname(sysctlVar, &sysctlBuffer, &len, NULL, 0) < 0) goto error; - /* sysfs base directory - * /sys/bus/vmbus/devices/7a08391f-f5a0-4ac0-9802-d13fd964f8df - * or on older kernel - * /sys/bus/vmbus/devices/vmbus_1 + /* pnpinfo: classid=f912ad6d-2b17-48ea-bd65-f927a61c7684 + * deviceid=d34b2567-b9b6-42b9-8778-0a4ec0b955bf */ - snprintf(dirname, sizeof(dirname), "%s/%s", - SYSFS_VMBUS_DEVICES, name); - - /* get device class */ - snprintf(filename, sizeof(filename), "%s/class_id", dirname); - if (parse_sysfs_uuid(filename, dev->class_id) < 0) + if (sysctlBuffer[0] == 'c' && sysctlBuffer[1] == 'l' && + sysctlBuffer[7] == '=') { + strncpy(classid, &sysctlBuffer[8], guid_len); + classid[guid_len] = '\0'; + } + if (parse_sysfs_uuid(classid, dev->class_id) < 0) goto error; /* skip non-network devices */ @@ -265,35 +235,23 @@ vmbus_scan_one(const char *name) return 0; } - /* get device id */ - snprintf(filename, sizeof(filename), "%s/device_id", dirname); - if (parse_sysfs_uuid(filename, dev->device_id) < 0) - goto error; - - /* get relid */ - snprintf(filename, sizeof(filename), "%s/id", dirname); - if (eal_parse_sysfs_value(filename, &tmp) < 0) + if (sysctlBuffer[45] == 'd' && sysctlBuffer[46] == 'e' && + sysctlBuffer[47] == 'v' && sysctlBuffer[53] == '=') { + strncpy(deviceid, &sysctlBuffer[54], guid_len); + deviceid[guid_len] = '\0'; + } + if (parse_sysfs_uuid(deviceid, dev->device_id) < 0) goto error; - dev->relid = tmp; - /* get monitor id */ - snprintf(filename, sizeof(filename), "%s/monitor_id", dirname); - if (eal_parse_sysfs_value(filename, &tmp) < 0) + if (!strcmp(name, "hv_uio")) + dev->uio_num = unit_num; + else + dev->uio_num = -1; + dev->device.bus = &rte_vmbus_bus.bus; + dev->device.numa_node = 0; + dev->device.name = strdup(deviceid); + if (!dev->device.name) goto error; - dev->monitor_id = tmp; - - /* get numa node (if present) */ - snprintf(filename, sizeof(filename), "%s/numa_node", - dirname); - - if (access(filename, R_OK) == 0) { - if (eal_parse_sysfs_value(filename, &tmp) < 0) - goto error; - dev->device.numa_node = tmp; - } else { - /* if no NUMA support, set default to 0 */ - dev->device.numa_node = SOCKET_ID_ANY; - } dev->device.devargs = vmbus_devargs_lookup(dev); @@ -332,31 +290,120 @@ vmbus_scan_one(const char *name) int rte_vmbus_scan(void) { - struct dirent *e; - DIR *dir; - - dir = opendir(SYSFS_VMBUS_DEVICES); - if (dir == NULL) { - if (errno == ENOENT) - return 0; + struct u_device udev; + struct u_businfo ubus; + int dev_idx, dev_ptr, name2oid[2], oid[CTL_MAXNAME + 12], error; + size_t oidlen, rlen, ub_size; + uintptr_t vmbus_handle = 0; + char *walker, *ep; + char name[16] = "hw.bus.devices"; + char *dd_name, *dd_desc, *dd_drivername, *dd_pnpinfo, *dd_location; + + /* + * devinfo FreeBSD APP logic to fetch all the VMBus devices + * using SYSCTLs + */ + name2oid[0] = 0; + name2oid[1] = 3; + oidlen = sizeof(oid); + error = sysctl(name2oid, 2, oid, &oidlen, name, strlen(name)); + if (error < 0) { + VMBUS_LOG(DEBUG, "can't find hw.bus.devices sysctl node"); + return -ENOENT; + } + oidlen /= sizeof(int); + if (oidlen > CTL_MAXNAME) { + VMBUS_LOG(DEBUG, "hw.bus.devices oid is too large"); + return -EINVAL; + } - VMBUS_LOG(ERR, "opendir %s failed: %s", - SYSFS_VMBUS_DEVICES, strerror(errno)); - return -1; + ub_size = sizeof(ubus); + if (sysctlbyname("hw.bus.info", &ubus, &ub_size, NULL, 0) != 0) { + VMBUS_LOG(DEBUG, "sysctlbyname(\"hw.bus.info\", ...) failed"); + return -EINVAL; + } + if ((ub_size != sizeof(ubus)) || + (ubus.ub_version != BUS_USER_VERSION)) { + VMBUS_LOG(DEBUG, + "kernel bus interface version mismatch: kernel %d expected %d", + ubus.ub_version, BUS_USER_VERSION); + return -EINVAL; } - while ((e = readdir(dir)) != NULL) { - if (e->d_name[0] == '.') - continue; + oid[oidlen++] = ubus.ub_generation; + dev_ptr = oidlen++; + + /* + * Scan devices. + * + * Stop after a fairly insane number to avoid death in the case + * of kernel corruption. + */ + + for (dev_idx = 0; dev_idx < 10000; dev_idx++) { + /* + * Get the device information. + */ + oid[dev_ptr] = dev_idx; + rlen = sizeof(udev); + error = sysctl(oid, oidlen, &udev, &rlen, NULL, 0); + if (error < 0) { + if (errno == ENOENT) /* end of list */ + break; + if (errno != EINVAL) /* gen count skip, restart */ + VMBUS_LOG(DEBUG, "sysctl hw.bus.devices.%d", + dev_idx); + return errno; + } + if (rlen != sizeof(udev)) { + VMBUS_LOG(DEBUG, + "sysctl returned wrong data %zd bytes instead of %zd", + rlen, sizeof(udev)); + return -EINVAL; + } - if (vmbus_scan_one(e->d_name) < 0) - goto error; + walker = udev.dv_fields; + ep = walker + sizeof(udev.dv_fields); + dd_name = NULL; + dd_desc = NULL; + dd_drivername = NULL; + dd_pnpinfo = NULL; + dd_location = NULL; +#define UNPACK(x) \ + do { \ + x = strdup(walker); \ + if (x == NULL) \ + return -ENOMEM; \ + if (walker + strnlen(walker, ep - walker) >= ep) \ + return -EINVAL; \ + walker += strlen(walker) + 1; \ + } while (0) + + UNPACK(dd_name); + UNPACK(dd_desc); + UNPACK(dd_drivername); + UNPACK(dd_pnpinfo); + UNPACK(dd_location); +#undef UNPACK + if (*dd_drivername && !(strcmp(dd_drivername, "vmbus"))) + vmbus_handle = udev.dv_handle; + + if (vmbus_handle && (vmbus_handle == udev.dv_parent) + && *dd_pnpinfo && *dd_name) { + unsigned int driver_len = 0, unit_num = 0; + char *endptr; + + driver_len = strlen(dd_drivername); + unit_num = strtoull(&dd_name[driver_len], &endptr, 10); + VMBUS_LOG(DEBUG, "Device name:%s, pnpinfo:%s", + dd_name, dd_pnpinfo); + + if (vmbus_scan_one(dd_drivername, unit_num) < 0) + goto error; + } } - closedir(dir); return 0; - error: - closedir(dir); return -1; } -- 2.30.2
All resource values are published by HV_UIO driver as sysctl key value pairs and they are read at a later point of the code flow Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_bus.c | 127 -------------------------- 1 file changed, 127 deletions(-) diff --git a/drivers/bus/vmbus/freebsd/vmbus_bus.c b/drivers/bus/vmbus/freebsd/vmbus_bus.c index 8eb428a154..9b23b1131c 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_bus.c +++ b/drivers/bus/vmbus/freebsd/vmbus_bus.c @@ -55,142 +55,15 @@ parse_sysfs_uuid(const char *filename, rte_uuid_t uu) return 0; } -static int -get_sysfs_string(const char *filename, char *buf, size_t buflen) -{ - char *cp; - FILE *f; - - f = fopen(filename, "r"); - if (f == NULL) { - VMBUS_LOG(ERR, "cannot open sysfs value %s:%s", - filename, strerror(errno)); - return -1; - } - - if (fgets(buf, buflen, f) == NULL) { - VMBUS_LOG(ERR, "cannot read sysfs value %s", - filename); - fclose(f); - return -1; - } - fclose(f); - - /* remove trailing newline */ - cp = memchr(buf, '\n', buflen); - if (cp) - *cp = '\0'; - - return 0; -} - -static int -vmbus_get_uio_dev(const struct rte_vmbus_device *dev, - char *dstbuf, size_t buflen) -{ - char dirname[PATH_MAX]; - unsigned int uio_num; - struct dirent *e; - DIR *dir; - - /* Assume recent kernel where uio is in uio/uioX */ - snprintf(dirname, sizeof(dirname), - SYSFS_VMBUS_DEVICES "/%s/uio", dev->device.name); - - dir = opendir(dirname); - if (dir == NULL) - return -1; /* Not a UIO device */ - - /* take the first file starting with "uio" */ - while ((e = readdir(dir)) != NULL) { - const int prefix_len = 3; - char *endptr; - - if (strncmp(e->d_name, "uio", prefix_len) != 0) - continue; - - /* try uio%d */ - errno = 0; - uio_num = strtoull(e->d_name + prefix_len, &endptr, 10); - if (errno == 0 && endptr != (e->d_name + prefix_len)) { - snprintf(dstbuf, buflen, "%s/uio%u", dirname, uio_num); - break; - } - } - closedir(dir); - - if (e == NULL) - return -1; - - return uio_num; -} - -/* Check map names with kernel names */ -static const char *map_names[VMBUS_MAX_RESOURCE] = { - [HV_TXRX_RING_MAP] = "txrx_rings", - [HV_INT_PAGE_MAP] = "int_page", - [HV_MON_PAGE_MAP] = "monitor_page", - [HV_RECV_BUF_MAP] = "recv:", - [HV_SEND_BUF_MAP] = "send:", -}; - - /* map the resources of a vmbus device in virtual memory */ int rte_vmbus_map_device(struct rte_vmbus_device *dev) { - char uioname[PATH_MAX], filename[PATH_MAX]; - char dirname[PATH_MAX], mapname[64]; - int i; - - dev->uio_num = vmbus_get_uio_dev(dev, uioname, sizeof(uioname)); if (dev->uio_num < 0) { VMBUS_LOG(DEBUG, "Not managed by UIO driver, skipped"); return 1; } - /* Extract resource value */ - for (i = 0; i < VMBUS_MAX_RESOURCE; i++) { - struct rte_mem_resource *res = &dev->resource[i]; - unsigned long len, gpad = 0; - char *cp; - - snprintf(dirname, sizeof(dirname), - "%s/maps/map%d", uioname, i); - - snprintf(filename, sizeof(filename), - "%s/name", dirname); - - if (get_sysfs_string(filename, mapname, sizeof(mapname)) < 0) { - VMBUS_LOG(ERR, "could not read %s", filename); - return -1; - } - - if (strncmp(map_names[i], mapname, strlen(map_names[i])) != 0) { - VMBUS_LOG(ERR, - "unexpected resource %s (expected %s)", - mapname, map_names[i]); - return -1; - } - - snprintf(filename, sizeof(filename), - "%s/size", dirname); - if (eal_parse_sysfs_value(filename, &len) < 0) { - VMBUS_LOG(ERR, - "could not read %s", filename); - return -1; - } - res->len = len; - - /* both send and receive buffers have gpad in name */ - cp = memchr(mapname, ':', sizeof(mapname)); - if (cp) - gpad = strtoul(cp+1, NULL, 0); - - /* put the GPAD value in physical address */ - res->phys_addr = gpad; - } - return vmbus_uio_map_resource(dev); } -- 2.30.2
The UIO device's relid, monitor id, etc values are retrieved using following sysctl variables: $ sysctl dev.hv_uio.0 dev.hv_uio.0.send_buf.gpadl: 925241 dev.hv_uio.0.send_buf.size: 16777216 dev.hv_uio.0.recv_buf.gpadl: 925240 dev.hv_uio.0.recv_buf.size: 32505856 dev.hv_uio.0.monitor_page.size: 4096 dev.hv_uio.0.int_page.size: 4096 Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 98 ++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 17 deletions(-) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index b52ca5bf1d..fdd37dac3a 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -10,6 +10,8 @@ #include <inttypes.h> #include <sys/stat.h> #include <sys/mman.h> +#include <sys/types.h> +#include <sys/sysctl.h> #include <rte_log.h> #include <rte_bus.h> @@ -24,8 +26,18 @@ /** Pathname of VMBUS devices directory. */ #define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" +const char *driver_name = "hv_uio"; static void *vmbus_map_addr; +/* Check map names with kernel names */ +static const char *map_names[VMBUS_MAX_RESOURCE] = { + [HV_TXRX_RING_MAP] = "txrx_rings", + [HV_INT_PAGE_MAP] = "int_page", + [HV_MON_PAGE_MAP] = "monitor_page", + [HV_RECV_BUF_MAP] = "recv_buf", + [HV_SEND_BUF_MAP] = "send_buf", +}; + /* Control interrupts */ void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff) { @@ -72,14 +84,75 @@ vmbus_uio_free_resource(struct rte_vmbus_device *dev, } } +static int +sysctl_get_vmbus_device_info(struct rte_vmbus_device *dev) +{ + char sysctlBuffer[PATH_MAX]; + char sysctlVar[PATH_MAX]; + size_t len = PATH_MAX, sysctl_len; + unsigned long tmp; + int i; + + snprintf(sysctlBuffer, len, "dev.%s.%d", driver_name, dev->uio_num); + + sysctl_len = sizeof(unsigned long); + /* get relid */ + snprintf(sysctlVar, len, "%s.channel.ch_id", sysctlBuffer); + if (sysctlbyname(sysctlVar, &tmp, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s", sysctlVar); + goto error; + } + dev->relid = tmp; + + /* get monitor id */ + snprintf(sysctlVar, len, "%s.channel.%u.monitor_id", sysctlBuffer, + dev->relid); + if (sysctlbyname(sysctlVar, &tmp, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s", sysctlVar); + goto error; + } + dev->monitor_id = tmp; + + /* Extract resource value */ + for (i = 0; i < VMBUS_MAX_RESOURCE; i++) { + struct rte_mem_resource *res = &dev->resource[i]; + unsigned long size, gpad = 0; + size_t sizelen = sizeof(len); + + snprintf(sysctlVar, sizeof(sysctlVar), "%s.%s.size", + sysctlBuffer, map_names[i]); + if (sysctlbyname(sysctlVar, &size, &sizelen, NULL, 0) < 0) { + VMBUS_LOG(ERR, + "could not read %s", sysctlVar); + goto error; + } + res->len = size; + + if (i == HV_RECV_BUF_MAP || i == HV_SEND_BUF_MAP) { + snprintf(sysctlVar, sizeof(sysctlVar), "%s.%s.gpadl", + sysctlBuffer, map_names[i]); + if (sysctlbyname(sysctlVar, &gpad, &sizelen, NULL, 0) < 0) { + VMBUS_LOG(ERR, + "could not read %s", sysctlVar); + goto error; + } + /* put the GPAD value in physical address */ + res->phys_addr = gpad; + } + } + return 0; +error: + return -1; +} + int vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, struct mapped_vmbus_resource **uio_res) { - char devname[PATH_MAX]; /* contains the /dev/uioX */ + char devname[PATH_MAX]; /* contains the /dev/hv_uioX */ /* save fd if in primary process */ - snprintf(devname, sizeof(devname), "/dev/uio%u", dev->uio_num); + snprintf(devname, sizeof(devname), "/dev/hv_uio%u", dev->uio_num); dev->intr_handle.fd = open(devname, O_RDWR); if (dev->intr_handle.fd < 0) { VMBUS_LOG(ERR, "Cannot open %s: %s", @@ -98,8 +171,10 @@ vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, strlcpy((*uio_res)->path, devname, PATH_MAX); rte_uuid_copy((*uio_res)->id, dev->device_id); - return 0; + if (sysctl_get_vmbus_device_info(dev) < 0) + goto error; + return 0; error: vmbus_uio_free_resource(dev, *uio_res); return -1; @@ -338,23 +413,12 @@ static int vmbus_uio_sysfs_read(const char *dir, const char *name, return ret; } -static bool vmbus_uio_ring_present(const struct rte_vmbus_device *dev, - uint32_t relid) -{ - char ring_path[PATH_MAX]; - - /* Check if kernel has subchannel sysfs files */ - snprintf(ring_path, sizeof(ring_path), - "%s/%s/channels/%u/ring", - SYSFS_VMBUS_DEVICES, dev->device.name, relid); - - return access(ring_path, R_OK|W_OK) == 0; -} - bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, const struct vmbus_channel *chan) { - return vmbus_uio_ring_present(dev, chan->relid); + RTE_SET_USED(dev); + RTE_SET_USED(chan); + return true; } static bool vmbus_isnew_subchannel(struct vmbus_channel *primary, -- 2.30.2
In FreeBSD, unlike Linux there is no sub-channel open callback that could be called by HV_UIO driver, upon their grant by the hypervisor. Thus, the PMD makes an IOCTL to the HV_UIO to open the granted sub-channels Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 31 +++++++++++++++++++++++++++ drivers/bus/vmbus/linux/vmbus_uio.c | 8 +++++++ drivers/bus/vmbus/private.h | 1 + drivers/bus/vmbus/rte_bus_vmbus.h | 10 +++++++++ drivers/bus/vmbus/version.map | 1 + drivers/bus/vmbus/vmbus_channel.c | 5 +++++ 6 files changed, 56 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index fdd37dac3a..022ac85302 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -12,6 +12,7 @@ #include <sys/mman.h> #include <sys/types.h> #include <sys/sysctl.h> +#include <sys/ioctl.h> #include <rte_log.h> #include <rte_bus.h> @@ -26,6 +27,9 @@ /** Pathname of VMBUS devices directory. */ #define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" +/* ioctl */ +#define HVIOOPENSUBCHAN _IOW('h', 14, uint32_t) + const char *driver_name = "hv_uio"; static void *vmbus_map_addr; @@ -515,3 +519,30 @@ int vmbus_uio_get_subchan(struct vmbus_channel *primary, closedir(chan_dir); return err; } + +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, uint32_t subchan) +{ + struct mapped_vmbus_resource *uio_res; + int fd, err = 0; + + uio_res = vmbus_uio_find_resource(dev); + if (!uio_res) { + VMBUS_LOG(ERR, "cannot find uio resource"); + return -EINVAL; + } + + fd = open(uio_res->path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + uio_res->path, strerror(errno)); + return -1; + } + + if (ioctl(fd, HVIOOPENSUBCHAN, &subchan)) { + VMBUS_LOG(ERR, "open subchan ioctl failed %s: %s", + uio_res->path, strerror(errno)); + err = -1; + } + close(fd); + return err; +} diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c b/drivers/bus/vmbus/linux/vmbus_uio.c index b52ca5bf1d..9e91ed9907 100644 --- a/drivers/bus/vmbus/linux/vmbus_uio.c +++ b/drivers/bus/vmbus/linux/vmbus_uio.c @@ -451,3 +451,11 @@ int vmbus_uio_get_subchan(struct vmbus_channel *primary, closedir(chan_dir); return err; } + +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, + uint32_t subchan) +{ + RTE_SET_USED(dev); + RTE_SET_USED(subchan); + return 0; +} diff --git a/drivers/bus/vmbus/private.h b/drivers/bus/vmbus/private.h index 528d60a42f..968f0b6f23 100644 --- a/drivers/bus/vmbus/private.h +++ b/drivers/bus/vmbus/private.h @@ -107,6 +107,7 @@ int vmbus_uio_get_subchan(struct vmbus_channel *primary, int vmbus_uio_map_rings(struct vmbus_channel *chan); int vmbus_uio_map_secondary_subchan(const struct rte_vmbus_device *dev, const struct vmbus_channel *chan); +int vmbus_uio_subchan_open(struct rte_vmbus_device *device, uint32_t subchan); void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen); diff --git a/drivers/bus/vmbus/rte_bus_vmbus.h b/drivers/bus/vmbus/rte_bus_vmbus.h index 4cf73ce815..42d13c5705 100644 --- a/drivers/bus/vmbus/rte_bus_vmbus.h +++ b/drivers/bus/vmbus/rte_bus_vmbus.h @@ -405,6 +405,16 @@ void rte_vmbus_chan_dump(FILE *f, const struct vmbus_channel *chan); */ void rte_vmbus_unregister(struct rte_vmbus_driver *driver); +/** + * Perform IOCTL to VMBUS device + * + * @param device + * A pointer to a rte_vmbus_device structure + * @param subchan + * Count of subchannels to open + */ +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan); + /** Helper for VMBUS device registration from driver instance */ #define RTE_PMD_REGISTER_VMBUS(nm, vmbus_drv) \ RTE_INIT(vmbusinitfn_ ##nm) \ diff --git a/drivers/bus/vmbus/version.map b/drivers/bus/vmbus/version.map index 3cadec7fae..3509d4fc14 100644 --- a/drivers/bus/vmbus/version.map +++ b/drivers/bus/vmbus/version.map @@ -23,6 +23,7 @@ DPDK_22 { rte_vmbus_subchan_open; rte_vmbus_unmap_device; rte_vmbus_unregister; + rte_vmbus_ioctl; local: *; }; diff --git a/drivers/bus/vmbus/vmbus_channel.c b/drivers/bus/vmbus/vmbus_channel.c index f67f1c438a..f53a1b6511 100644 --- a/drivers/bus/vmbus/vmbus_channel.c +++ b/drivers/bus/vmbus/vmbus_channel.c @@ -367,6 +367,11 @@ int rte_vmbus_max_channels(const struct rte_vmbus_device *device) return 1; } +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan) +{ + return vmbus_uio_subchan_open(device, subchan); +} + /* Setup secondary channel */ int rte_vmbus_subchan_open(struct vmbus_channel *primary, struct vmbus_channel **new_chan) -- 2.30.2
On Linux, the request does nothing, while on FreeBSD the HV_UIO kernel driver receives the request and opens the subchannels on PMDs behalf. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/hn_ethdev.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c index 9e2a405973..6f9053c4d5 100644 --- a/drivers/net/netvsc/hn_ethdev.c +++ b/drivers/net/netvsc/hn_ethdev.c @@ -515,6 +515,10 @@ static int hn_subchan_configure(struct hn_data *hv, if (err) return err; + err = rte_vmbus_ioctl(hv->vmbus, subchan); + if (err) + return err; + while (subchan > 0) { struct vmbus_channel *new_sc; uint16_t chn_index; -- 2.30.2
Using sysctl, the resource values of subchannels are obtained and an mmap request to made to their buffers. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 181 ++++++++++---------------- 1 file changed, 66 insertions(+), 115 deletions(-) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index 022ac85302..14b858f39d 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -24,8 +24,11 @@ #include "private.h" -/** Pathname of VMBUS devices directory. */ -#define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" +/* Macros to distinguish mmap request + * [7-0] - Device memory region + * [15-8]- Sub-channel id + */ +#define UH_SUBCHAN_MASK_SHIFT 8 /* ioctl */ #define HVIOOPENSUBCHAN _IOW('h', 14, uint32_t) @@ -274,19 +277,17 @@ static int vmbus_uio_map_primary(struct vmbus_channel *chan, } static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev, - const struct vmbus_channel *chan, + struct vmbus_channel *chan, void **ring_buf, uint32_t *ring_size) { char ring_path[PATH_MAX]; - size_t file_size; - struct stat sb; + size_t size; void *mapaddr; + off_t offset; int fd; snprintf(ring_path, sizeof(ring_path), - "%s/%s/channels/%u/ring", - SYSFS_VMBUS_DEVICES, dev->device.name, - chan->relid); + "/dev/hv_uio%d", dev->uio_num); fd = open(ring_path, O_RDWR); if (fd < 0) { @@ -295,33 +296,21 @@ static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev, return -errno; } - if (fstat(fd, &sb) < 0) { - VMBUS_LOG(ERR, "Cannot state %s: %s", - ring_path, strerror(errno)); - close(fd); - return -errno; - } - file_size = sb.st_size; - - if (file_size == 0 || (file_size & (rte_mem_page_size() - 1))) { - VMBUS_LOG(ERR, "incorrect size %s: %zu", - ring_path, file_size); - - close(fd); - return -EINVAL; - } + /* subchannel rings are of the same size as primary */ + size = dev->resource[HV_TXRX_RING_MAP].len; + offset = (chan->relid << UH_SUBCHAN_MASK_SHIFT) * PAGE_SIZE; mapaddr = vmbus_map_resource(vmbus_map_addr, fd, - 0, file_size, 0); + offset, size, 0); close(fd); if (mapaddr == MAP_FAILED) return -EIO; - *ring_size = file_size / 2; + *ring_size = size / 2; *ring_buf = mapaddr; - vmbus_map_addr = RTE_PTR_ADD(mapaddr, file_size); + vmbus_map_addr = RTE_PTR_ADD(mapaddr, size); return 0; } @@ -336,9 +325,7 @@ vmbus_uio_map_secondary_subchan(const struct rte_vmbus_device *dev, int fd; snprintf(ring_path, sizeof(ring_path), - "%s/%s/channels/%u/ring", - SYSFS_VMBUS_DEVICES, dev->device.name, - chan->relid); + "/dev/ring%d", dev->uio_num); ring_buf = br->vbr; ring_size = br->dsize + sizeof(struct vmbus_bufring); @@ -391,32 +378,6 @@ int vmbus_uio_map_rings(struct vmbus_channel *chan) return 0; } -static int vmbus_uio_sysfs_read(const char *dir, const char *name, - unsigned long *val, unsigned long max_range) -{ - char path[PATH_MAX]; - FILE *f; - int ret; - - snprintf(path, sizeof(path), "%s/%s", dir, name); - f = fopen(path, "r"); - if (!f) { - VMBUS_LOG(ERR, "can't open %s:%s", - path, strerror(errno)); - return -errno; - } - - if (fscanf(f, "%lu", val) != 1) - ret = -EIO; - else if (*val > max_range) - ret = -ERANGE; - else - ret = 0; - fclose(f); - - return ret; -} - bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, const struct vmbus_channel *chan) { @@ -426,7 +387,7 @@ bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, } static bool vmbus_isnew_subchannel(struct vmbus_channel *primary, - unsigned long id) + uint16_t id) { const struct vmbus_channel *c; @@ -441,83 +402,73 @@ int vmbus_uio_get_subchan(struct vmbus_channel *primary, struct vmbus_channel **subchan) { const struct rte_vmbus_device *dev = primary->device; - char chan_path[PATH_MAX], subchan_path[PATH_MAX]; - struct dirent *ent; - DIR *chan_dir; + char sysctlBuffer[PATH_MAX], sysctlVar[PATH_MAX]; + size_t len = PATH_MAX, sysctl_len; + /* nr_schan, relid, subid & monid datatype must match kernel's for sysctl */ + uint32_t relid, subid, nr_schan, i; + uint8_t monid; int err; - snprintf(chan_path, sizeof(chan_path), - "%s/%s/channels", - SYSFS_VMBUS_DEVICES, dev->device.name); - - chan_dir = opendir(chan_path); - if (!chan_dir) { - VMBUS_LOG(ERR, "cannot open %s: %s", - chan_path, strerror(errno)); - return -errno; + /* get no. of sub-channels opened by hv_uio + * dev.hv_uio.0.subchan_cnt + */ + snprintf(sysctlVar, len, "dev.%s.%d.subchan_cnt", driver_name, + dev->uio_num); + sysctl_len = sizeof(nr_schan); + if (sysctlbyname(sysctlVar, &nr_schan, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctlVar, + strerror(errno)); + return -1; } - while ((ent = readdir(chan_dir))) { - unsigned long relid, subid, monid; - char *endp; - - if (ent->d_name[0] == '.') - continue; - - errno = 0; - relid = strtoul(ent->d_name, &endp, 0); - if (*endp || errno != 0 || relid > UINT16_MAX) { - VMBUS_LOG(NOTICE, "not a valid channel relid: %s", - ent->d_name); - continue; - } - - if (!vmbus_isnew_subchannel(primary, relid)) { - VMBUS_LOG(DEBUG, "skip already found channel: %lu", - relid); - continue; + /* dev.hv_uio.0.channel.14.sub */ + snprintf(sysctlBuffer, len, "dev.%s.%d.channel.%u.sub", driver_name, + dev->uio_num, primary->relid); + for (i = 1; i <= nr_schan; i++) { + /* get relid */ + snprintf(sysctlVar, len, "%s.%u.chanid", sysctlBuffer, i); + sysctl_len = sizeof(relid); + if (sysctlbyname(sysctlVar, &relid, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctlVar, + strerror(errno)); + goto error; } - if (!vmbus_uio_ring_present(dev, relid)) { - VMBUS_LOG(DEBUG, "ring mmap not found (yet) for: %lu", - relid); + if (!vmbus_isnew_subchannel(primary, (uint16_t)relid)) { + VMBUS_LOG(DEBUG, "skip already found channel: %u", + relid); continue; } - snprintf(subchan_path, sizeof(subchan_path), "%s/%lu", - chan_path, relid); - err = vmbus_uio_sysfs_read(subchan_path, "subchannel_id", - &subid, UINT16_MAX); - if (err) { - VMBUS_LOG(NOTICE, "no subchannel_id in %s:%s", - subchan_path, strerror(-err)); - goto fail; + /* get sub-channel id */ + snprintf(sysctlVar, len, "%s.%u.ch_subidx", sysctlBuffer, i); + sysctl_len = sizeof(subid); + if (sysctlbyname(sysctlVar, &subid, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctlVar, + strerror(errno)); + goto error; } - if (subid == 0) - continue; /* skip primary channel */ - - err = vmbus_uio_sysfs_read(subchan_path, "monitor_id", - &monid, UINT8_MAX); - if (err) { - VMBUS_LOG(NOTICE, "no monitor_id in %s:%s", - subchan_path, strerror(-err)); - goto fail; + /* get monitor id */ + snprintf(sysctlVar, len, "%s.%u.monitor_id", sysctlBuffer, i); + sysctl_len = sizeof(monid); + if (sysctlbyname(sysctlVar, &monid, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctlVar, + strerror(errno)); + goto error; } - err = vmbus_chan_create(dev, relid, subid, monid, subchan); + err = vmbus_chan_create(dev, (uint16_t)relid, (uint16_t)subid, + monid, subchan); if (err) { VMBUS_LOG(ERR, "subchannel setup failed"); - goto fail; + return err; } break; } - closedir(chan_dir); - - return (ent == NULL) ? -ENOENT : 0; -fail: - closedir(chan_dir); - return err; + return 0; +error: + return -1; } int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, uint32_t subchan) -- 2.30.2
Event monitoring is not yet supported on FreeBSD, hence moving it the OS specific files Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/freebsd/hn_os.c | 14 ++++++++++++++ drivers/net/netvsc/hn_ethdev.c | 7 +++---- drivers/net/netvsc/hn_os.h | 6 ++++++ drivers/net/netvsc/linux/hn_os.c | 19 +++++++++++++++++++ 4 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 drivers/net/netvsc/freebsd/hn_os.c create mode 100644 drivers/net/netvsc/hn_os.h create mode 100644 drivers/net/netvsc/linux/hn_os.c diff --git a/drivers/net/netvsc/freebsd/hn_os.c b/drivers/net/netvsc/freebsd/hn_os.c new file mode 100644 index 0000000000..3bd67e06c8 --- /dev/null +++ b/drivers/net/netvsc/freebsd/hn_os.c @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2021 Microsoft Corporation + */ + +#include <stdio.h> + +#include "hn_logs.h" +#include "hn_os.h" + +int eth_hn_os_dev_event(void) +{ + PMD_DRV_LOG(DEBUG, "rte_dev_event_monitor_start not supported on FreeBSD"); + return 0; +} diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c index 6f9053c4d5..61d7d3daeb 100644 --- a/drivers/net/netvsc/hn_ethdev.c +++ b/drivers/net/netvsc/hn_ethdev.c @@ -39,6 +39,7 @@ #include "hn_rndis.h" #include "hn_nvs.h" #include "ndis.h" +#include "hn_os.h" #define HN_TX_OFFLOAD_CAPS (DEV_TX_OFFLOAD_IPV4_CKSUM | \ DEV_TX_OFFLOAD_TCP_CKSUM | \ @@ -1244,11 +1245,9 @@ static int eth_hn_probe(struct rte_vmbus_driver *drv __rte_unused, PMD_INIT_FUNC_TRACE(); - ret = rte_dev_event_monitor_start(); - if (ret) { - PMD_DRV_LOG(ERR, "Failed to start device event monitoring"); + ret = eth_hn_os_dev_event(); + if (ret) return ret; - } eth_dev = eth_dev_vmbus_allocate(dev, sizeof(struct hn_data)); if (!eth_dev) diff --git a/drivers/net/netvsc/hn_os.h b/drivers/net/netvsc/hn_os.h new file mode 100644 index 0000000000..618c53cdcd --- /dev/null +++ b/drivers/net/netvsc/hn_os.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2009-2021 Microsoft Corp. + * All rights reserved. + */ + +int eth_hn_os_dev_event(void); diff --git a/drivers/net/netvsc/linux/hn_os.c b/drivers/net/netvsc/linux/hn_os.c new file mode 100644 index 0000000000..862dc190c1 --- /dev/null +++ b/drivers/net/netvsc/linux/hn_os.c @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2021 Microsoft Corporation + */ + +#include <rte_ethdev.h> + +#include "hn_logs.h" +#include "hn_os.h" + +int eth_hn_os_dev_event(void) +{ + int ret; + + ret = rte_dev_event_monitor_start(); + if (ret) + PMD_DRV_LOG(ERR, "Failed to start device event monitoring"); + + return ret; +} -- 2.30.2
Moved netvsc_hotplug_retry to respective OS dir as it contains OS dependent code. For Linux, it is copied as is and for FreeBSD it is not supported yet. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/freebsd/hn_os.c | 8 +++ drivers/net/netvsc/hn_ethdev.c | 84 --------------------------- drivers/net/netvsc/hn_os.h | 2 + drivers/net/netvsc/linux/hn_os.c | 92 ++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 84 deletions(-) diff --git a/drivers/net/netvsc/freebsd/hn_os.c b/drivers/net/netvsc/freebsd/hn_os.c index 3bd67e06c8..2ba4c32a76 100644 --- a/drivers/net/netvsc/freebsd/hn_os.c +++ b/drivers/net/netvsc/freebsd/hn_os.c @@ -4,6 +4,8 @@ #include <stdio.h> +#include <rte_common.h> + #include "hn_logs.h" #include "hn_os.h" @@ -12,3 +14,9 @@ int eth_hn_os_dev_event(void) PMD_DRV_LOG(DEBUG, "rte_dev_event_monitor_start not supported on FreeBSD"); return 0; } + +void netvsc_hotplug_retry(void *args) +{ + RTE_SET_USED(args); + return; +} diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c index 61d7d3daeb..c299b98a6c 100644 --- a/drivers/net/netvsc/hn_ethdev.c +++ b/drivers/net/netvsc/hn_ethdev.c @@ -57,9 +57,6 @@ #define NETVSC_ARG_TXBREAK "tx_copybreak" #define NETVSC_ARG_RX_EXTMBUF_ENABLE "rx_extmbuf_enable" -/* The max number of retry when hot adding a VF device */ -#define NETVSC_MAX_HOTADD_RETRY 10 - struct hn_xstats_name_off { char name[RTE_ETH_XSTATS_NAME_SIZE]; unsigned int offset; @@ -556,87 +553,6 @@ static int hn_subchan_configure(struct hn_data *hv, return err; } -static void netvsc_hotplug_retry(void *args) -{ - int ret; - struct hn_data *hv = args; - struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id]; - struct rte_devargs *d = &hv->devargs; - char buf[256]; - - DIR *di; - struct dirent *dir; - struct ifreq req; - struct rte_ether_addr eth_addr; - int s; - - PMD_DRV_LOG(DEBUG, "%s: retry count %d", - __func__, hv->eal_hot_plug_retry); - - if (hv->eal_hot_plug_retry++ > NETVSC_MAX_HOTADD_RETRY) - return; - - snprintf(buf, sizeof(buf), "/sys/bus/pci/devices/%s/net", d->name); - di = opendir(buf); - if (!di) { - PMD_DRV_LOG(DEBUG, "%s: can't open directory %s, " - "retrying in 1 second", __func__, buf); - goto retry; - } - - while ((dir = readdir(di))) { - /* Skip . and .. directories */ - if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) - continue; - - /* trying to get mac address if this is a network device*/ - s = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); - if (s == -1) { - PMD_DRV_LOG(ERR, "Failed to create socket errno %d", - errno); - break; - } - strlcpy(req.ifr_name, dir->d_name, sizeof(req.ifr_name)); - ret = ioctl(s, SIOCGIFHWADDR, &req); - close(s); - if (ret == -1) { - PMD_DRV_LOG(ERR, - "Failed to send SIOCGIFHWADDR for device %s", - dir->d_name); - break; - } - if (req.ifr_hwaddr.sa_family != ARPHRD_ETHER) { - closedir(di); - return; - } - memcpy(eth_addr.addr_bytes, req.ifr_hwaddr.sa_data, - RTE_DIM(eth_addr.addr_bytes)); - - if (rte_is_same_ether_addr(ð_addr, dev->data->mac_addrs)) { - PMD_DRV_LOG(NOTICE, - "Found matching MAC address, adding device %s network name %s", - d->name, dir->d_name); - ret = rte_eal_hotplug_add(d->bus->name, d->name, - d->args); - if (ret) { - PMD_DRV_LOG(ERR, - "Failed to add PCI device %s", - d->name); - break; - } - } - /* When the code reaches here, we either have already added - * the device, or its MAC address did not match. - */ - closedir(di); - return; - } - closedir(di); -retry: - /* The device is still being initialized, retry after 1 second */ - rte_eal_alarm_set(1000000, netvsc_hotplug_retry, hv); -} - static void netvsc_hotadd_callback(const char *device_name, enum rte_dev_event_type type, void *arg) diff --git a/drivers/net/netvsc/hn_os.h b/drivers/net/netvsc/hn_os.h index 618c53cdcd..1fb7292b17 100644 --- a/drivers/net/netvsc/hn_os.h +++ b/drivers/net/netvsc/hn_os.h @@ -4,3 +4,5 @@ */ int eth_hn_os_dev_event(void); + +void netvsc_hotplug_retry(void *args); diff --git a/drivers/net/netvsc/linux/hn_os.c b/drivers/net/netvsc/linux/hn_os.c index 862dc190c1..9c2f4cd7a8 100644 --- a/drivers/net/netvsc/linux/hn_os.c +++ b/drivers/net/netvsc/linux/hn_os.c @@ -2,11 +2,22 @@ * Copyright(c) 2016-2021 Microsoft Corporation */ +#include <unistd.h> +#include <dirent.h> +#include <net/if.h> +#include <net/if_arp.h> +#include <sys/ioctl.h> + #include <rte_ethdev.h> +#include <rte_alarm.h> #include "hn_logs.h" +#include "hn_var.h" #include "hn_os.h" +/* The max number of retry when hot adding a VF device */ +#define NETVSC_MAX_HOTADD_RETRY 10 + int eth_hn_os_dev_event(void) { int ret; @@ -17,3 +28,84 @@ int eth_hn_os_dev_event(void) return ret; } + +void netvsc_hotplug_retry(void *args) +{ + int ret; + struct hn_data *hv = args; + struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id]; + struct rte_devargs *d = &hv->devargs; + char buf[256]; + + DIR *di; + struct dirent *dir; + struct ifreq req; + struct rte_ether_addr eth_addr; + int s; + + PMD_DRV_LOG(DEBUG, "%s: retry count %d", + __func__, hv->eal_hot_plug_retry); + + if (hv->eal_hot_plug_retry++ > NETVSC_MAX_HOTADD_RETRY) + return; + + snprintf(buf, sizeof(buf), "/sys/bus/pci/devices/%s/net", d->name); + di = opendir(buf); + if (!di) { + PMD_DRV_LOG(DEBUG, "%s: can't open directory %s, " + "retrying in 1 second", __func__, buf); + goto retry; + } + + while ((dir = readdir(di))) { + /* Skip . and .. directories */ + if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) + continue; + + /* trying to get mac address if this is a network device*/ + s = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); + if (s == -1) { + PMD_DRV_LOG(ERR, "Failed to create socket errno %d", + errno); + break; + } + strlcpy(req.ifr_name, dir->d_name, sizeof(req.ifr_name)); + ret = ioctl(s, SIOCGIFHWADDR, &req); + close(s); + if (ret == -1) { + PMD_DRV_LOG(ERR, + "Failed to send SIOCGIFHWADDR for device %s", + dir->d_name); + break; + } + if (req.ifr_hwaddr.sa_family != ARPHRD_ETHER) { + closedir(di); + return; + } + memcpy(eth_addr.addr_bytes, req.ifr_hwaddr.sa_data, + RTE_DIM(eth_addr.addr_bytes)); + + if (rte_is_same_ether_addr(ð_addr, dev->data->mac_addrs)) { + PMD_DRV_LOG(NOTICE, + "Found matching MAC address, adding device %s network name %s", + d->name, dir->d_name); + ret = rte_eal_hotplug_add(d->bus->name, d->name, + d->args); + if (ret) { + PMD_DRV_LOG(ERR, + "Failed to add PCI device %s", + d->name); + break; + } + } + /* When the code reaches here, we either have already added + * the device, or its MAC address did not match. + */ + closedir(di); + return; + } + closedir(di); +retry: + /* The device is still being initialized, retry after 1 second */ + rte_eal_alarm_set(1000000, netvsc_hotplug_retry, hv); +} -- 2.30.2
add meson support for FreeBSD OS Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/meson.build | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build index 3892cbf67f..8fbe144fcd 100644 --- a/drivers/bus/vmbus/meson.build +++ b/drivers/bus/vmbus/meson.build @@ -20,7 +20,11 @@ if is_linux sources += files('linux/vmbus_bus.c', 'linux/vmbus_uio.c') includes += include_directories('linux') +elif is_freebsd + sources += files('freebsd/vmbus_bus.c', + 'freebsd/vmbus_uio.c') + includes += include_directories('freebsd') else build = false - reason = 'only supported on Linux' + reason = 'only supported on Linux & FreeBSD' endif -- 2.30.2
add meson support for FreeBSD Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/freebsd/meson.build | 6 ++++++ drivers/net/netvsc/linux/meson.build | 6 ++++++ drivers/net/netvsc/meson.build | 4 ++++ 3 files changed, 16 insertions(+) create mode 100644 drivers/net/netvsc/freebsd/meson.build create mode 100644 drivers/net/netvsc/linux/meson.build diff --git a/drivers/net/netvsc/freebsd/meson.build b/drivers/net/netvsc/freebsd/meson.build new file mode 100644 index 0000000000..78f824f701 --- /dev/null +++ b/drivers/net/netvsc/freebsd/meson.build @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Microsoft Corporation + +sources += files( + 'hn_os.c', +) diff --git a/drivers/net/netvsc/linux/meson.build b/drivers/net/netvsc/linux/meson.build new file mode 100644 index 0000000000..78f824f701 --- /dev/null +++ b/drivers/net/netvsc/linux/meson.build @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Microsoft Corporation + +sources += files( + 'hn_os.c', +) diff --git a/drivers/net/netvsc/meson.build b/drivers/net/netvsc/meson.build index f74b941f65..9dafa8ac23 100644 --- a/drivers/net/netvsc/meson.build +++ b/drivers/net/netvsc/meson.build @@ -1,6 +1,8 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Microsoft Corporation +includes += include_directories(exec_env) + if is_windows build = false reason = 'not supported on Windows' @@ -18,3 +20,5 @@ sources = files( ) deps += ['bus_vmbus' ] + +subdir(exec_env) -- 2.30.2
> Subject: [PATCH 05/11] bus/vmbus: open subchannels > > [You don't often get email from srikanth.k@oneconvergence.com. Learn > why this is important at http://aka.ms/LearnAboutSenderIdentification.] > > In FreeBSD, unlike Linux there is no sub-channel open callback that could be > called by HV_UIO driver, upon their grant by the hypervisor. > Thus, the PMD makes an IOCTL to the HV_UIO to open the granted sub- > channels > > Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> > Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> > Signed-off-by: Anand Thulasiram <avelu@juniper.net> > --- > drivers/bus/vmbus/freebsd/vmbus_uio.c | 31 > +++++++++++++++++++++++++++ > drivers/bus/vmbus/linux/vmbus_uio.c | 8 +++++++ > drivers/bus/vmbus/private.h | 1 + > drivers/bus/vmbus/rte_bus_vmbus.h | 10 +++++++++ > drivers/bus/vmbus/version.map | 1 + > drivers/bus/vmbus/vmbus_channel.c | 5 +++++ > 6 files changed, 56 insertions(+) > > diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c > b/drivers/bus/vmbus/freebsd/vmbus_uio.c > index fdd37dac3a..022ac85302 100644 > --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c > +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c > @@ -12,6 +12,7 @@ > #include <sys/mman.h> > #include <sys/types.h> > #include <sys/sysctl.h> > +#include <sys/ioctl.h> > > #include <rte_log.h> > #include <rte_bus.h> > @@ -26,6 +27,9 @@ > /** Pathname of VMBUS devices directory. */ #define > SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" > > +/* ioctl */ > +#define HVIOOPENSUBCHAN _IOW('h', 14, uint32_t) > + > const char *driver_name = "hv_uio"; > static void *vmbus_map_addr; > > @@ -515,3 +519,30 @@ int vmbus_uio_get_subchan(struct vmbus_channel > *primary, > closedir(chan_dir); > return err; > } > + > +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, uint32_t > +subchan) { > + struct mapped_vmbus_resource *uio_res; > + int fd, err = 0; > + > + uio_res = vmbus_uio_find_resource(dev); > + if (!uio_res) { > + VMBUS_LOG(ERR, "cannot find uio resource"); > + return -EINVAL; > + } > + > + fd = open(uio_res->path, O_RDWR); > + if (fd < 0) { > + VMBUS_LOG(ERR, "Cannot open %s: %s", > + uio_res->path, strerror(errno)); > + return -1; > + } > + > + if (ioctl(fd, HVIOOPENSUBCHAN, &subchan)) { > + VMBUS_LOG(ERR, "open subchan ioctl failed %s: %s", > + uio_res->path, strerror(errno)); > + err = -1; > + } > + close(fd); > + return err; > +} > diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c > b/drivers/bus/vmbus/linux/vmbus_uio.c > index b52ca5bf1d..9e91ed9907 100644 > --- a/drivers/bus/vmbus/linux/vmbus_uio.c > +++ b/drivers/bus/vmbus/linux/vmbus_uio.c > @@ -451,3 +451,11 @@ int vmbus_uio_get_subchan(struct vmbus_channel > *primary, > closedir(chan_dir); > return err; > } > + > +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, > + uint32_t subchan) { > + RTE_SET_USED(dev); > + RTE_SET_USED(subchan); > + return 0; > +} If this function ever gets called in Linux and it doesn't do anything, should it return a failure? > diff --git a/drivers/bus/vmbus/private.h b/drivers/bus/vmbus/private.h > index 528d60a42f..968f0b6f23 100644 > --- a/drivers/bus/vmbus/private.h > +++ b/drivers/bus/vmbus/private.h > @@ -107,6 +107,7 @@ int vmbus_uio_get_subchan(struct vmbus_channel > *primary, int vmbus_uio_map_rings(struct vmbus_channel *chan); int > vmbus_uio_map_secondary_subchan(const struct rte_vmbus_device *dev, > const struct vmbus_channel *chan); > +int vmbus_uio_subchan_open(struct rte_vmbus_device *device, uint32_t > +subchan); > > void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen); > > diff --git a/drivers/bus/vmbus/rte_bus_vmbus.h > b/drivers/bus/vmbus/rte_bus_vmbus.h > index 4cf73ce815..42d13c5705 100644 > --- a/drivers/bus/vmbus/rte_bus_vmbus.h > +++ b/drivers/bus/vmbus/rte_bus_vmbus.h > @@ -405,6 +405,16 @@ void rte_vmbus_chan_dump(FILE *f, const struct > vmbus_channel *chan); > */ > void rte_vmbus_unregister(struct rte_vmbus_driver *driver); > > +/** > + * Perform IOCTL to VMBUS device > + * > + * @param device > + * A pointer to a rte_vmbus_device structure > + * @param subchan > + * Count of subchannels to open > + */ > +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan); > + > /** Helper for VMBUS device registration from driver instance */ > #define RTE_PMD_REGISTER_VMBUS(nm, vmbus_drv) \ > RTE_INIT(vmbusinitfn_ ##nm) \ > diff --git a/drivers/bus/vmbus/version.map > b/drivers/bus/vmbus/version.map index 3cadec7fae..3509d4fc14 100644 > --- a/drivers/bus/vmbus/version.map > +++ b/drivers/bus/vmbus/version.map > @@ -23,6 +23,7 @@ DPDK_22 { > rte_vmbus_subchan_open; > rte_vmbus_unmap_device; > rte_vmbus_unregister; > + rte_vmbus_ioctl; > > local: *; > }; > diff --git a/drivers/bus/vmbus/vmbus_channel.c > b/drivers/bus/vmbus/vmbus_channel.c > index f67f1c438a..f53a1b6511 100644 > --- a/drivers/bus/vmbus/vmbus_channel.c > +++ b/drivers/bus/vmbus/vmbus_channel.c > @@ -367,6 +367,11 @@ int rte_vmbus_max_channels(const struct > rte_vmbus_device *device) > return 1; > } > > +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan) > +{ > + return vmbus_uio_subchan_open(device, subchan); } > + > /* Setup secondary channel */ > int rte_vmbus_subchan_open(struct vmbus_channel *primary, > struct vmbus_channel **new_chan) > -- > 2.30.2
> Subject: [PATCH 06/11] net/netvsc: request HV_UIO to open sub-channels > > [You don't often get email from srikanth.k@oneconvergence.com. Learn > why this is important at http://aka.ms/LearnAboutSenderIdentification.] > > On Linux, the request does nothing, while on FreeBSD the HV_UIO kernel > driver receives the request and opens the subchannels on PMDs behalf. > > Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> > Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> > Signed-off-by: Anand Thulasiram <avelu@juniper.net> > --- > drivers/net/netvsc/hn_ethdev.c | 4 ++++ > 1 file changed, 4 insertions(+) > > diff --git a/drivers/net/netvsc/hn_ethdev.c > b/drivers/net/netvsc/hn_ethdev.c index 9e2a405973..6f9053c4d5 100644 > --- a/drivers/net/netvsc/hn_ethdev.c > +++ b/drivers/net/netvsc/hn_ethdev.c > @@ -515,6 +515,10 @@ static int hn_subchan_configure(struct hn_data *hv, > if (err) > return err; > > + err = rte_vmbus_ioctl(hv->vmbus, subchan); > + if (err) > + return err; > + Now I see why you return 0 for Linux. Can you add a comment to say this function is not used and should always succeed in Linux? > while (subchan > 0) { > struct vmbus_channel *new_sc; > uint16_t chn_index; > -- > 2.30.2
> Subject: [PATCH 00/11] add FreeBSD support to VMBUS & NetVSC PMDs
>
> [You don't often get email from srikanth.k@oneconvergence.com. Learn
> why this is important at http://aka.ms/LearnAboutSenderIdentification.]
>
> This patch series adds support to VMBUS & NetVSC PMDs run on FreeBSD
>
> Srikanth Kaka (11):
> bus/vmbus: stub for FreeBSD support
> bus/vmbus: scan and get the network device
> bus/vmbus: handle mapping of device resources
> bus/vmbus: get device resource values using sysctl
> bus/vmbus: open subchannels
> net/netvsc: request HV_UIO to open sub-channels
> bus/vmbus: map the subchannel resources
> net/netvsc: moving event monitoring support
> net/netvsc: moving hotplug retry to OS dir
> bus/vmbus: add meson suport for FreeBSD OS
> net/netvsc: add meson support for FreeBSD
>
> drivers/bus/vmbus/freebsd/vmbus_bus.c | 296 +++++++++++++++
> drivers/bus/vmbus/freebsd/vmbus_uio.c | 499
> +++++++++++++++++++++++++
> drivers/bus/vmbus/linux/vmbus_uio.c | 8 +
> drivers/bus/vmbus/meson.build | 6 +-
> drivers/bus/vmbus/private.h | 1 +
> drivers/bus/vmbus/rte_bus_vmbus.h | 10 +
> drivers/bus/vmbus/version.map | 1 +
> drivers/bus/vmbus/vmbus_channel.c | 5 +
> drivers/net/netvsc/freebsd/hn_os.c | 22 ++
> drivers/net/netvsc/freebsd/meson.build | 6 +
> drivers/net/netvsc/hn_ethdev.c | 95 +----
> drivers/net/netvsc/hn_os.h | 8 +
> drivers/net/netvsc/linux/hn_os.c | 111 ++++++
> drivers/net/netvsc/linux/meson.build | 6 +
> drivers/net/netvsc/meson.build | 4 +
> 15 files changed, 989 insertions(+), 89 deletions(-) create mode 100644
> drivers/bus/vmbus/freebsd/vmbus_bus.c
> create mode 100644 drivers/bus/vmbus/freebsd/vmbus_uio.c
> create mode 100644 drivers/net/netvsc/freebsd/hn_os.c
> create mode 100644 drivers/net/netvsc/freebsd/meson.build
> create mode 100644 drivers/net/netvsc/hn_os.h create mode 100644
> drivers/net/netvsc/linux/hn_os.c create mode 100644
> drivers/net/netvsc/linux/meson.build
>
> --
> 2.30.2
Can you also update ./MAINTAINERS for FreeBSD?
Long
Sure. I will add the comment in v2.
Regards,
Srikanth
On Fri, Oct 1, 2021 at 4:49 AM Long Li <longli@microsoft.com> wrote:
> > Subject: [PATCH 06/11] net/netvsc: request HV_UIO to open sub-channels
> >
> > [You don't often get email from srikanth.k@oneconvergence.com. Learn
> > why this is important at http://aka.ms/LearnAboutSenderIdentification.]
> >
> > On Linux, the request does nothing, while on FreeBSD the HV_UIO kernel
> > driver receives the request and opens the subchannels on PMDs behalf.
> >
> > Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com>
> > Signed-off-by: Vag Singh <vag.singh@oneconvergence.com>
> > Signed-off-by: Anand Thulasiram <avelu@juniper.net>
> > ---
> > drivers/net/netvsc/hn_ethdev.c | 4 ++++
> > 1 file changed, 4 insertions(+)
> >
> > diff --git a/drivers/net/netvsc/hn_ethdev.c
> > b/drivers/net/netvsc/hn_ethdev.c index 9e2a405973..6f9053c4d5 100644
> > --- a/drivers/net/netvsc/hn_ethdev.c
> > +++ b/drivers/net/netvsc/hn_ethdev.c
> > @@ -515,6 +515,10 @@ static int hn_subchan_configure(struct hn_data *hv,
> > if (err)
> > return err;
> >
> > + err = rte_vmbus_ioctl(hv->vmbus, subchan);
> > + if (err)
> > + return err;
> > +
>
> Now I see why you return 0 for Linux. Can you add a comment to say this
> function is not used and should always succeed in Linux?
>
> > while (subchan > 0) {
> > struct vmbus_channel *new_sc;
> > uint16_t chn_index;
> > --
> > 2.30.2
>
>
Using sysctl, all the devices on the VMBUS are identified by the PMD. On finding the Network device's device id, it is added to VMBUS dev list. v2 - replaced strncpy with memcpy - replaced malloc.h with stdlib.h Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_bus.c | 241 +++++++++++++++----------- drivers/net/netvsc/hn_rxtx.c | 2 +- 2 files changed, 145 insertions(+), 98 deletions(-) diff --git a/drivers/bus/vmbus/freebsd/vmbus_bus.c b/drivers/bus/vmbus/freebsd/vmbus_bus.c index 3c924eee14..47b85c18c0 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_bus.c +++ b/drivers/bus/vmbus/freebsd/vmbus_bus.c @@ -22,8 +22,9 @@ #include "eal_filesystem.h" #include "private.h" -/** Pathname of VMBUS devices directory. */ -#define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" +#include <sys/bus.h> +#include <sys/types.h> +#include <sys/sysctl.h> /* * GUID associated with network devices @@ -39,44 +40,15 @@ static const rte_uuid_t vmbus_nic_uuid = { extern struct rte_vmbus_bus rte_vmbus_bus; -/* Read sysfs file to get UUID */ +/* Parse UUID. Caller must pass NULL terminated string */ static int parse_sysfs_uuid(const char *filename, rte_uuid_t uu) { - char buf[BUFSIZ]; - char *cp, *in = buf; - FILE *f; - - f = fopen(filename, "r"); - if (f == NULL) { - VMBUS_LOG(ERR, "cannot open sysfs value %s: %s", - filename, strerror(errno)); - return -1; - } - - if (fgets(buf, sizeof(buf), f) == NULL) { - VMBUS_LOG(ERR, "cannot read sysfs value %s", - filename); - fclose(f); - return -1; - } - fclose(f); - - cp = strchr(buf, '\n'); - if (cp) - *cp = '\0'; - - /* strip { } notation */ - if (buf[0] == '{') { - in = buf + 1; - cp = strchr(in, '}'); - if (cp) - *cp = '\0'; - } + char in[BUFSIZ]; + memcpy(in, filename, BUFSIZ); if (rte_uuid_parse(in, uu) < 0) { - VMBUS_LOG(ERR, "%s %s not a valid UUID", - filename, buf); + VMBUS_LOG(ERR, "%s not a valid UUID", in); return -1; } @@ -228,35 +200,33 @@ rte_vmbus_unmap_device(struct rte_vmbus_device *dev) vmbus_uio_unmap_resource(dev); } -/* Scan one vmbus sysfs entry, and fill the devices list from it. */ +/* Scan one vmbus entry, and fill the devices list from it. */ static int -vmbus_scan_one(const char *name) +vmbus_scan_one(const char *name, unsigned int unit_num) { struct rte_vmbus_device *dev, *dev2; - char filename[PATH_MAX]; - char dirname[PATH_MAX]; - unsigned long tmp; + char sysctlBuffer[PATH_MAX], sysctlVar[PATH_MAX]; + size_t guid_len = 36, len = PATH_MAX; + char classid[guid_len + 1], deviceid[guid_len + 1]; dev = calloc(1, sizeof(*dev)); if (dev == NULL) return -1; - dev->device.bus = &rte_vmbus_bus.bus; - dev->device.name = strdup(name); - if (!dev->device.name) + /* get class id and device id */ + snprintf(sysctlVar, len, "dev.%s.%u.%%pnpinfo", name, unit_num); + if (sysctlbyname(sysctlVar, &sysctlBuffer, &len, NULL, 0) < 0) goto error; - /* sysfs base directory - * /sys/bus/vmbus/devices/7a08391f-f5a0-4ac0-9802-d13fd964f8df - * or on older kernel - * /sys/bus/vmbus/devices/vmbus_1 + /* pnpinfo: classid=f912ad6d-2b17-48ea-bd65-f927a61c7684 + * deviceid=d34b2567-b9b6-42b9-8778-0a4ec0b955bf */ - snprintf(dirname, sizeof(dirname), "%s/%s", - SYSFS_VMBUS_DEVICES, name); - - /* get device class */ - snprintf(filename, sizeof(filename), "%s/class_id", dirname); - if (parse_sysfs_uuid(filename, dev->class_id) < 0) + if (sysctlBuffer[0] == 'c' && sysctlBuffer[1] == 'l' && + sysctlBuffer[7] == '=') { + memcpy(classid, &sysctlBuffer[8], guid_len); + classid[guid_len] = '\0'; + } + if (parse_sysfs_uuid(classid, dev->class_id) < 0) goto error; /* skip non-network devices */ @@ -265,35 +235,23 @@ vmbus_scan_one(const char *name) return 0; } - /* get device id */ - snprintf(filename, sizeof(filename), "%s/device_id", dirname); - if (parse_sysfs_uuid(filename, dev->device_id) < 0) - goto error; - - /* get relid */ - snprintf(filename, sizeof(filename), "%s/id", dirname); - if (eal_parse_sysfs_value(filename, &tmp) < 0) + if (sysctlBuffer[45] == 'd' && sysctlBuffer[46] == 'e' && + sysctlBuffer[47] == 'v' && sysctlBuffer[53] == '=') { + memcpy(deviceid, &sysctlBuffer[54], guid_len); + deviceid[guid_len] = '\0'; + } + if (parse_sysfs_uuid(deviceid, dev->device_id) < 0) goto error; - dev->relid = tmp; - /* get monitor id */ - snprintf(filename, sizeof(filename), "%s/monitor_id", dirname); - if (eal_parse_sysfs_value(filename, &tmp) < 0) + if (!strcmp(name, "hv_uio")) + dev->uio_num = unit_num; + else + dev->uio_num = -1; + dev->device.bus = &rte_vmbus_bus.bus; + dev->device.numa_node = 0; + dev->device.name = strdup(deviceid); + if (!dev->device.name) goto error; - dev->monitor_id = tmp; - - /* get numa node (if present) */ - snprintf(filename, sizeof(filename), "%s/numa_node", - dirname); - - if (access(filename, R_OK) == 0) { - if (eal_parse_sysfs_value(filename, &tmp) < 0) - goto error; - dev->device.numa_node = tmp; - } else { - /* if no NUMA support, set default to 0 */ - dev->device.numa_node = SOCKET_ID_ANY; - } dev->device.devargs = vmbus_devargs_lookup(dev); @@ -332,31 +290,120 @@ vmbus_scan_one(const char *name) int rte_vmbus_scan(void) { - struct dirent *e; - DIR *dir; - - dir = opendir(SYSFS_VMBUS_DEVICES); - if (dir == NULL) { - if (errno == ENOENT) - return 0; + struct u_device udev; + struct u_businfo ubus; + int dev_idx, dev_ptr, name2oid[2], oid[CTL_MAXNAME + 12], error; + size_t oidlen, rlen, ub_size; + uintptr_t vmbus_handle = 0; + char *walker, *ep; + char name[16] = "hw.bus.devices"; + char *dd_name, *dd_desc, *dd_drivername, *dd_pnpinfo, *dd_location; + + /* + * devinfo FreeBSD APP logic to fetch all the VMBus devices + * using SYSCTLs + */ + name2oid[0] = 0; + name2oid[1] = 3; + oidlen = sizeof(oid); + error = sysctl(name2oid, 2, oid, &oidlen, name, strlen(name)); + if (error < 0) { + VMBUS_LOG(DEBUG, "can't find hw.bus.devices sysctl node"); + return -ENOENT; + } + oidlen /= sizeof(int); + if (oidlen > CTL_MAXNAME) { + VMBUS_LOG(DEBUG, "hw.bus.devices oid is too large"); + return -EINVAL; + } - VMBUS_LOG(ERR, "opendir %s failed: %s", - SYSFS_VMBUS_DEVICES, strerror(errno)); - return -1; + ub_size = sizeof(ubus); + if (sysctlbyname("hw.bus.info", &ubus, &ub_size, NULL, 0) != 0) { + VMBUS_LOG(DEBUG, "sysctlbyname(\"hw.bus.info\", ...) failed"); + return -EINVAL; + } + if ((ub_size != sizeof(ubus)) || + (ubus.ub_version != BUS_USER_VERSION)) { + VMBUS_LOG(DEBUG, + "kernel bus interface version mismatch: kernel %d expected %d", + ubus.ub_version, BUS_USER_VERSION); + return -EINVAL; } - while ((e = readdir(dir)) != NULL) { - if (e->d_name[0] == '.') - continue; + oid[oidlen++] = ubus.ub_generation; + dev_ptr = oidlen++; + + /* + * Scan devices. + * + * Stop after a fairly insane number to avoid death in the case + * of kernel corruption. + */ + + for (dev_idx = 0; dev_idx < 10000; dev_idx++) { + /* + * Get the device information. + */ + oid[dev_ptr] = dev_idx; + rlen = sizeof(udev); + error = sysctl(oid, oidlen, &udev, &rlen, NULL, 0); + if (error < 0) { + if (errno == ENOENT) /* end of list */ + break; + if (errno != EINVAL) /* gen count skip, restart */ + VMBUS_LOG(DEBUG, "sysctl hw.bus.devices.%d", + dev_idx); + return errno; + } + if (rlen != sizeof(udev)) { + VMBUS_LOG(DEBUG, + "sysctl returned wrong data %zd bytes instead of %zd", + rlen, sizeof(udev)); + return -EINVAL; + } - if (vmbus_scan_one(e->d_name) < 0) - goto error; + walker = udev.dv_fields; + ep = walker + sizeof(udev.dv_fields); + dd_name = NULL; + dd_desc = NULL; + dd_drivername = NULL; + dd_pnpinfo = NULL; + dd_location = NULL; +#define UNPACK(x) \ + do { \ + x = strdup(walker); \ + if (x == NULL) \ + return -ENOMEM; \ + if (walker + strnlen(walker, ep - walker) >= ep) \ + return -EINVAL; \ + walker += strlen(walker) + 1; \ + } while (0) + + UNPACK(dd_name); + UNPACK(dd_desc); + UNPACK(dd_drivername); + UNPACK(dd_pnpinfo); + UNPACK(dd_location); +#undef UNPACK + if (*dd_drivername && !(strcmp(dd_drivername, "vmbus"))) + vmbus_handle = udev.dv_handle; + + if (vmbus_handle && (vmbus_handle == udev.dv_parent) + && *dd_pnpinfo && *dd_name) { + unsigned int driver_len = 0, unit_num = 0; + char *endptr; + + driver_len = strlen(dd_drivername); + unit_num = strtoull(&dd_name[driver_len], &endptr, 10); + VMBUS_LOG(DEBUG, "Device name:%s, pnpinfo:%s", + dd_name, dd_pnpinfo); + + if (vmbus_scan_one(dd_drivername, unit_num) < 0) + goto error; + } } - closedir(dir); return 0; - error: - closedir(dir); return -1; } diff --git a/drivers/net/netvsc/hn_rxtx.c b/drivers/net/netvsc/hn_rxtx.c index c6bf7cc132..f306f2f3f0 100644 --- a/drivers/net/netvsc/hn_rxtx.c +++ b/drivers/net/netvsc/hn_rxtx.c @@ -10,7 +10,7 @@ #include <errno.h> #include <unistd.h> #include <strings.h> -#include <malloc.h> +#include <stdlib.h> #include <rte_ethdev.h> #include <rte_memcpy.h> -- 2.30.1
In FreeBSD, unlike Linux there is no sub-channel open callback that could be called by HV_UIO driver, upon their grant by the hypervisor. Thus, the PMD makes an IOCTL to the HV_UIO to open the granted sub-channels v2 - Added comment in linux/vmbus_uio.c Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 31 +++++++++++++++++++++++++++ drivers/bus/vmbus/linux/vmbus_uio.c | 12 +++++++++++ drivers/bus/vmbus/private.h | 1 + drivers/bus/vmbus/rte_bus_vmbus.h | 10 +++++++++ drivers/bus/vmbus/version.map | 1 + drivers/bus/vmbus/vmbus_channel.c | 5 +++++ 6 files changed, 60 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index fdd37dac3a..022ac85302 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -12,6 +12,7 @@ #include <sys/mman.h> #include <sys/types.h> #include <sys/sysctl.h> +#include <sys/ioctl.h> #include <rte_log.h> #include <rte_bus.h> @@ -26,6 +27,9 @@ /** Pathname of VMBUS devices directory. */ #define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" +/* ioctl */ +#define HVIOOPENSUBCHAN _IOW('h', 14, uint32_t) + const char *driver_name = "hv_uio"; static void *vmbus_map_addr; @@ -515,3 +519,30 @@ int vmbus_uio_get_subchan(struct vmbus_channel *primary, closedir(chan_dir); return err; } + +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, uint32_t subchan) +{ + struct mapped_vmbus_resource *uio_res; + int fd, err = 0; + + uio_res = vmbus_uio_find_resource(dev); + if (!uio_res) { + VMBUS_LOG(ERR, "cannot find uio resource"); + return -EINVAL; + } + + fd = open(uio_res->path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + uio_res->path, strerror(errno)); + return -1; + } + + if (ioctl(fd, HVIOOPENSUBCHAN, &subchan)) { + VMBUS_LOG(ERR, "open subchan ioctl failed %s: %s", + uio_res->path, strerror(errno)); + err = -1; + } + close(fd); + return err; +} diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c b/drivers/bus/vmbus/linux/vmbus_uio.c index b52ca5bf1d..29e889347a 100644 --- a/drivers/bus/vmbus/linux/vmbus_uio.c +++ b/drivers/bus/vmbus/linux/vmbus_uio.c @@ -451,3 +451,15 @@ int vmbus_uio_get_subchan(struct vmbus_channel *primary, closedir(chan_dir); return err; } + +/* + * This is a stub function and it should always succeed. + * The Linux UIO kernel driver opens the subchannels implicitly. + */ +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, + uint32_t subchan) +{ + RTE_SET_USED(dev); + RTE_SET_USED(subchan); + return 0; +} diff --git a/drivers/bus/vmbus/private.h b/drivers/bus/vmbus/private.h index 528d60a42f..968f0b6f23 100644 --- a/drivers/bus/vmbus/private.h +++ b/drivers/bus/vmbus/private.h @@ -107,6 +107,7 @@ int vmbus_uio_get_subchan(struct vmbus_channel *primary, int vmbus_uio_map_rings(struct vmbus_channel *chan); int vmbus_uio_map_secondary_subchan(const struct rte_vmbus_device *dev, const struct vmbus_channel *chan); +int vmbus_uio_subchan_open(struct rte_vmbus_device *device, uint32_t subchan); void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen); diff --git a/drivers/bus/vmbus/rte_bus_vmbus.h b/drivers/bus/vmbus/rte_bus_vmbus.h index 6bcff66468..60c5812706 100644 --- a/drivers/bus/vmbus/rte_bus_vmbus.h +++ b/drivers/bus/vmbus/rte_bus_vmbus.h @@ -404,6 +404,16 @@ void rte_vmbus_chan_dump(FILE *f, const struct vmbus_channel *chan); */ void rte_vmbus_unregister(struct rte_vmbus_driver *driver); +/** + * Perform IOCTL to VMBUS device + * + * @param device + * A pointer to a rte_vmbus_device structure + * @param subchan + * Count of subchannels to open + */ +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan); + /** Helper for VMBUS device registration from driver instance */ #define RTE_PMD_REGISTER_VMBUS(nm, vmbus_drv) \ RTE_INIT(vmbusinitfn_ ##nm) \ diff --git a/drivers/bus/vmbus/version.map b/drivers/bus/vmbus/version.map index 3cadec7fae..3509d4fc14 100644 --- a/drivers/bus/vmbus/version.map +++ b/drivers/bus/vmbus/version.map @@ -23,6 +23,7 @@ DPDK_22 { rte_vmbus_subchan_open; rte_vmbus_unmap_device; rte_vmbus_unregister; + rte_vmbus_ioctl; local: *; }; diff --git a/drivers/bus/vmbus/vmbus_channel.c b/drivers/bus/vmbus/vmbus_channel.c index f67f1c438a..f53a1b6511 100644 --- a/drivers/bus/vmbus/vmbus_channel.c +++ b/drivers/bus/vmbus/vmbus_channel.c @@ -367,6 +367,11 @@ int rte_vmbus_max_channels(const struct rte_vmbus_device *device) return 1; } +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan) +{ + return vmbus_uio_subchan_open(device, subchan); +} + /* Setup secondary channel */ int rte_vmbus_subchan_open(struct vmbus_channel *primary, struct vmbus_channel **new_chan) -- 2.30.1
add meson support for FreeBSD v2 - moved include statement to after OS detection - updated ./MAINTAINERS file Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- MAINTAINERS | 2 ++ drivers/net/netvsc/freebsd/meson.build | 6 ++++++ drivers/net/netvsc/linux/meson.build | 6 ++++++ drivers/net/netvsc/meson.build | 4 ++++ 4 files changed, 18 insertions(+) create mode 100644 drivers/net/netvsc/freebsd/meson.build create mode 100644 drivers/net/netvsc/linux/meson.build diff --git a/MAINTAINERS b/MAINTAINERS index 278e5b3226..930d789945 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -552,6 +552,7 @@ F: drivers/bus/vdev/ VMBUS bus driver M: Stephen Hemminger <sthemmin@microsoft.com> M: Long Li <longli@microsoft.com> +M: Srikanth Kaka <srikanth.k@oneconvergence.com> F: drivers/bus/vmbus/ @@ -824,6 +825,7 @@ F: doc/guides/nics/vdev_netvsc.rst Microsoft Hyper-V netvsc M: Stephen Hemminger <sthemmin@microsoft.com> M: Long Li <longli@microsoft.com> +M: Srikanth Kaka <srikanth.k@oneconvergence.com> F: drivers/net/netvsc/ F: doc/guides/nics/netvsc.rst F: doc/guides/nics/features/netvsc.ini diff --git a/drivers/net/netvsc/freebsd/meson.build b/drivers/net/netvsc/freebsd/meson.build new file mode 100644 index 0000000000..78f824f701 --- /dev/null +++ b/drivers/net/netvsc/freebsd/meson.build @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Microsoft Corporation + +sources += files( + 'hn_os.c', +) diff --git a/drivers/net/netvsc/linux/meson.build b/drivers/net/netvsc/linux/meson.build new file mode 100644 index 0000000000..78f824f701 --- /dev/null +++ b/drivers/net/netvsc/linux/meson.build @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Microsoft Corporation + +sources += files( + 'hn_os.c', +) diff --git a/drivers/net/netvsc/meson.build b/drivers/net/netvsc/meson.build index f74b941f65..6c9f5cfc56 100644 --- a/drivers/net/netvsc/meson.build +++ b/drivers/net/netvsc/meson.build @@ -7,6 +7,8 @@ if is_windows subdir_done() endif +includes += include_directories(exec_env) + build = dpdk_conf.has('RTE_BUS_VMBUS') reason = 'missing dependency, DPDK VMBus driver' sources = files( @@ -18,3 +20,5 @@ sources = files( ) deps += ['bus_vmbus' ] + +subdir(exec_env) -- 2.30.1
> From: Srikanth Kaka <srikanth.k@oneconvergence.com> > Sent: Monday, September 27, 2021 6:42 AM > To: Stephen Hemminger <sthemmin@microsoft.com>; Long Li > <longli@microsoft.com> > Cc: dev@dpdk.org; Vag Singh <vag.singh@oneconvergence.com>; Anand > Thulasiram <avelu@juniper.net>; Srikanth Kaka > <srikanth.k@oneconvergence.com> > Subject: [PATCH 01/11] bus/vmbus: stub for FreeBSD support > > [You don't often get email from srikanth.k@oneconvergence.com. Learn why > this is important at http://aka.ms/LearnAboutSenderIdentification.] > > These files are a copy of their Linux equivalents. > They will be ported to FreeBSD. > > Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> > Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> > Signed-off-by: Anand Thulasiram <avelu@juniper.net> Reviewed-by: Long Li <longli@microsoft.com> > --- > drivers/bus/vmbus/freebsd/vmbus_bus.c | 376 +++++++++++++++++++++ > drivers/bus/vmbus/freebsd/vmbus_uio.c | 453 ++++++++++++++++++++++++++ > 2 files changed, 829 insertions(+) > create mode 100644 drivers/bus/vmbus/freebsd/vmbus_bus.c > create mode 100644 drivers/bus/vmbus/freebsd/vmbus_uio.c > > diff --git a/drivers/bus/vmbus/freebsd/vmbus_bus.c > b/drivers/bus/vmbus/freebsd/vmbus_bus.c > new file mode 100644 > index 0000000000..3c924eee14 > --- /dev/null > +++ b/drivers/bus/vmbus/freebsd/vmbus_bus.c > @@ -0,0 +1,376 @@ > +/* SPDX-License-Identifier: BSD-3-Clause > + * Copyright (c) 2018, Microsoft Corporation. > + * All Rights Reserved. > + */ > + > +#include <string.h> > +#include <unistd.h> > +#include <dirent.h> > +#include <fcntl.h> > +#include <sys/mman.h> > +#include <sys/stat.h> > + > +#include <rte_eal.h> > +#include <rte_uuid.h> > +#include <rte_tailq.h> > +#include <rte_log.h> > +#include <rte_devargs.h> > +#include <rte_memory.h> > +#include <rte_malloc.h> > +#include <rte_bus_vmbus.h> > + > +#include "eal_filesystem.h" > +#include "private.h" > + > +/** Pathname of VMBUS devices directory. */ #define SYSFS_VMBUS_DEVICES > +"/sys/bus/vmbus/devices" > + > +/* > + * GUID associated with network devices > + * {f8615163-df3e-46c5-913f-f2d2f965ed0e} > + */ > +static const rte_uuid_t vmbus_nic_uuid = { > + 0xf8, 0x61, 0x51, 0x63, > + 0xdf, 0x3e, > + 0x46, 0xc5, > + 0x91, 0x3f, > + 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0xe }; > + > +extern struct rte_vmbus_bus rte_vmbus_bus; > + > +/* Read sysfs file to get UUID */ > +static int > +parse_sysfs_uuid(const char *filename, rte_uuid_t uu) { > + char buf[BUFSIZ]; > + char *cp, *in = buf; > + FILE *f; > + > + f = fopen(filename, "r"); > + if (f == NULL) { > + VMBUS_LOG(ERR, "cannot open sysfs value %s: %s", > + filename, strerror(errno)); > + return -1; > + } > + > + if (fgets(buf, sizeof(buf), f) == NULL) { > + VMBUS_LOG(ERR, "cannot read sysfs value %s", > + filename); > + fclose(f); > + return -1; > + } > + fclose(f); > + > + cp = strchr(buf, '\n'); > + if (cp) > + *cp = '\0'; > + > + /* strip { } notation */ > + if (buf[0] == '{') { > + in = buf + 1; > + cp = strchr(in, '}'); > + if (cp) > + *cp = '\0'; > + } > + > + if (rte_uuid_parse(in, uu) < 0) { > + VMBUS_LOG(ERR, "%s %s not a valid UUID", > + filename, buf); > + return -1; > + } > + > + return 0; > +} > + > +static int > +get_sysfs_string(const char *filename, char *buf, size_t buflen) { > + char *cp; > + FILE *f; > + > + f = fopen(filename, "r"); > + if (f == NULL) { > + VMBUS_LOG(ERR, "cannot open sysfs value %s:%s", > + filename, strerror(errno)); > + return -1; > + } > + > + if (fgets(buf, buflen, f) == NULL) { > + VMBUS_LOG(ERR, "cannot read sysfs value %s", > + filename); > + fclose(f); > + return -1; > + } > + fclose(f); > + > + /* remove trailing newline */ > + cp = memchr(buf, '\n', buflen); > + if (cp) > + *cp = '\0'; > + > + return 0; > +} > + > +static int > +vmbus_get_uio_dev(const struct rte_vmbus_device *dev, > + char *dstbuf, size_t buflen) { > + char dirname[PATH_MAX]; > + unsigned int uio_num; > + struct dirent *e; > + DIR *dir; > + > + /* Assume recent kernel where uio is in uio/uioX */ > + snprintf(dirname, sizeof(dirname), > + SYSFS_VMBUS_DEVICES "/%s/uio", dev->device.name); > + > + dir = opendir(dirname); > + if (dir == NULL) > + return -1; /* Not a UIO device */ > + > + /* take the first file starting with "uio" */ > + while ((e = readdir(dir)) != NULL) { > + const int prefix_len = 3; > + char *endptr; > + > + if (strncmp(e->d_name, "uio", prefix_len) != 0) > + continue; > + > + /* try uio%d */ > + errno = 0; > + uio_num = strtoull(e->d_name + prefix_len, &endptr, 10); > + if (errno == 0 && endptr != (e->d_name + prefix_len)) { > + snprintf(dstbuf, buflen, "%s/uio%u", dirname, uio_num); > + break; > + } > + } > + closedir(dir); > + > + if (e == NULL) > + return -1; > + > + return uio_num; > +} > + > +/* Check map names with kernel names */ static const char > +*map_names[VMBUS_MAX_RESOURCE] = { > + [HV_TXRX_RING_MAP] = "txrx_rings", > + [HV_INT_PAGE_MAP] = "int_page", > + [HV_MON_PAGE_MAP] = "monitor_page", > + [HV_RECV_BUF_MAP] = "recv:", > + [HV_SEND_BUF_MAP] = "send:", > +}; > + > + > +/* map the resources of a vmbus device in virtual memory */ int > +rte_vmbus_map_device(struct rte_vmbus_device *dev) { > + char uioname[PATH_MAX], filename[PATH_MAX]; > + char dirname[PATH_MAX], mapname[64]; > + int i; > + > + dev->uio_num = vmbus_get_uio_dev(dev, uioname, sizeof(uioname)); > + if (dev->uio_num < 0) { > + VMBUS_LOG(DEBUG, "Not managed by UIO driver, skipped"); > + return 1; > + } > + > + /* Extract resource value */ > + for (i = 0; i < VMBUS_MAX_RESOURCE; i++) { > + struct rte_mem_resource *res = &dev->resource[i]; > + unsigned long len, gpad = 0; > + char *cp; > + > + snprintf(dirname, sizeof(dirname), > + "%s/maps/map%d", uioname, i); > + > + snprintf(filename, sizeof(filename), > + "%s/name", dirname); > + > + if (get_sysfs_string(filename, mapname, sizeof(mapname)) < 0) { > + VMBUS_LOG(ERR, "could not read %s", filename); > + return -1; > + } > + > + if (strncmp(map_names[i], mapname, strlen(map_names[i])) != 0) { > + VMBUS_LOG(ERR, > + "unexpected resource %s (expected %s)", > + mapname, map_names[i]); > + return -1; > + } > + > + snprintf(filename, sizeof(filename), > + "%s/size", dirname); > + if (eal_parse_sysfs_value(filename, &len) < 0) { > + VMBUS_LOG(ERR, > + "could not read %s", filename); > + return -1; > + } > + res->len = len; > + > + /* both send and receive buffers have gpad in name */ > + cp = memchr(mapname, ':', sizeof(mapname)); > + if (cp) > + gpad = strtoul(cp+1, NULL, 0); > + > + /* put the GPAD value in physical address */ > + res->phys_addr = gpad; > + } > + > + return vmbus_uio_map_resource(dev); } > + > +void > +rte_vmbus_unmap_device(struct rte_vmbus_device *dev) { > + vmbus_uio_unmap_resource(dev); > +} > + > +/* Scan one vmbus sysfs entry, and fill the devices list from it. */ > +static int vmbus_scan_one(const char *name) { > + struct rte_vmbus_device *dev, *dev2; > + char filename[PATH_MAX]; > + char dirname[PATH_MAX]; > + unsigned long tmp; > + > + dev = calloc(1, sizeof(*dev)); > + if (dev == NULL) > + return -1; > + > + dev->device.bus = &rte_vmbus_bus.bus; > + dev->device.name = strdup(name); > + if (!dev->device.name) > + goto error; > + > + /* sysfs base directory > + * /sys/bus/vmbus/devices/7a08391f-f5a0-4ac0-9802-d13fd964f8df > + * or on older kernel > + * /sys/bus/vmbus/devices/vmbus_1 > + */ > + snprintf(dirname, sizeof(dirname), "%s/%s", > + SYSFS_VMBUS_DEVICES, name); > + > + /* get device class */ > + snprintf(filename, sizeof(filename), "%s/class_id", dirname); > + if (parse_sysfs_uuid(filename, dev->class_id) < 0) > + goto error; > + > + /* skip non-network devices */ > + if (rte_uuid_compare(dev->class_id, vmbus_nic_uuid) != 0) { > + free(dev); > + return 0; > + } > + > + /* get device id */ > + snprintf(filename, sizeof(filename), "%s/device_id", dirname); > + if (parse_sysfs_uuid(filename, dev->device_id) < 0) > + goto error; > + > + /* get relid */ > + snprintf(filename, sizeof(filename), "%s/id", dirname); > + if (eal_parse_sysfs_value(filename, &tmp) < 0) > + goto error; > + dev->relid = tmp; > + > + /* get monitor id */ > + snprintf(filename, sizeof(filename), "%s/monitor_id", dirname); > + if (eal_parse_sysfs_value(filename, &tmp) < 0) > + goto error; > + dev->monitor_id = tmp; > + > + /* get numa node (if present) */ > + snprintf(filename, sizeof(filename), "%s/numa_node", > + dirname); > + > + if (access(filename, R_OK) == 0) { > + if (eal_parse_sysfs_value(filename, &tmp) < 0) > + goto error; > + dev->device.numa_node = tmp; > + } else { > + /* if no NUMA support, set default to 0 */ > + dev->device.numa_node = SOCKET_ID_ANY; > + } > + > + dev->device.devargs = vmbus_devargs_lookup(dev); > + > + /* device is valid, add in list (sorted) */ > + VMBUS_LOG(DEBUG, "Adding vmbus device %s", name); > + > + TAILQ_FOREACH(dev2, &rte_vmbus_bus.device_list, next) { > + int ret; > + > + ret = rte_uuid_compare(dev->device_id, dev2->device_id); > + if (ret > 0) > + continue; > + > + if (ret < 0) { > + vmbus_insert_device(dev2, dev); > + } else { /* already registered */ > + VMBUS_LOG(NOTICE, > + "%s already registered", name); > + free(dev); > + } > + return 0; > + } > + > + vmbus_add_device(dev); > + return 0; > +error: > + VMBUS_LOG(DEBUG, "failed"); > + > + free(dev); > + return -1; > +} > + > +/* > + * Scan the content of the vmbus, and the devices in the devices list > +*/ int > +rte_vmbus_scan(void) > +{ > + struct dirent *e; > + DIR *dir; > + > + dir = opendir(SYSFS_VMBUS_DEVICES); > + if (dir == NULL) { > + if (errno == ENOENT) > + return 0; > + > + VMBUS_LOG(ERR, "opendir %s failed: %s", > + SYSFS_VMBUS_DEVICES, strerror(errno)); > + return -1; > + } > + > + while ((e = readdir(dir)) != NULL) { > + if (e->d_name[0] == '.') > + continue; > + > + if (vmbus_scan_one(e->d_name) < 0) > + goto error; > + } > + closedir(dir); > + return 0; > + > +error: > + closedir(dir); > + return -1; > +} > + > +void rte_vmbus_irq_mask(struct rte_vmbus_device *device) { > + vmbus_uio_irq_control(device, 1); } > + > +void rte_vmbus_irq_unmask(struct rte_vmbus_device *device) { > + vmbus_uio_irq_control(device, 0); } > + > +int rte_vmbus_irq_read(struct rte_vmbus_device *device) { > + return vmbus_uio_irq_read(device); } > diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c > b/drivers/bus/vmbus/freebsd/vmbus_uio.c > new file mode 100644 > index 0000000000..b52ca5bf1d > --- /dev/null > +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c > @@ -0,0 +1,453 @@ > +/* SPDX-License-Identifier: BSD-3-Clause > + * Copyright (c) 2018, Microsoft Corporation. > + * All Rights Reserved. > + */ > + > +#include <string.h> > +#include <unistd.h> > +#include <fcntl.h> > +#include <dirent.h> > +#include <inttypes.h> > +#include <sys/stat.h> > +#include <sys/mman.h> > + > +#include <rte_log.h> > +#include <rte_bus.h> > +#include <rte_memory.h> > +#include <rte_common.h> > +#include <rte_malloc.h> > +#include <rte_bus_vmbus.h> > +#include <rte_string_fns.h> > + > +#include "private.h" > + > +/** Pathname of VMBUS devices directory. */ #define SYSFS_VMBUS_DEVICES > +"/sys/bus/vmbus/devices" > + > +static void *vmbus_map_addr; > + > +/* Control interrupts */ > +void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff) > +{ > + if (write(dev->intr_handle.fd, &onoff, sizeof(onoff)) < 0) { > + VMBUS_LOG(ERR, "cannot write to %d:%s", > + dev->intr_handle.fd, strerror(errno)); > + } > +} > + > +int vmbus_uio_irq_read(struct rte_vmbus_device *dev) { > + int32_t count; > + int cc; > + > + cc = read(dev->intr_handle.fd, &count, sizeof(count)); > + if (cc < (int)sizeof(count)) { > + if (cc < 0) { > + VMBUS_LOG(ERR, "IRQ read failed %s", > + strerror(errno)); > + return -errno; > + } > + VMBUS_LOG(ERR, "can't read IRQ count"); > + return -EINVAL; > + } > + > + return count; > +} > + > +void > +vmbus_uio_free_resource(struct rte_vmbus_device *dev, > + struct mapped_vmbus_resource *uio_res) { > + rte_free(uio_res); > + > + if (dev->intr_handle.uio_cfg_fd >= 0) { > + close(dev->intr_handle.uio_cfg_fd); > + dev->intr_handle.uio_cfg_fd = -1; > + } > + > + if (dev->intr_handle.fd >= 0) { > + close(dev->intr_handle.fd); > + dev->intr_handle.fd = -1; > + dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; > + } > +} > + > +int > +vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, > + struct mapped_vmbus_resource **uio_res) { > + char devname[PATH_MAX]; /* contains the /dev/uioX */ > + > + /* save fd if in primary process */ > + snprintf(devname, sizeof(devname), "/dev/uio%u", dev->uio_num); > + dev->intr_handle.fd = open(devname, O_RDWR); > + if (dev->intr_handle.fd < 0) { > + VMBUS_LOG(ERR, "Cannot open %s: %s", > + devname, strerror(errno)); > + goto error; > + } > + dev->intr_handle.type = RTE_INTR_HANDLE_UIO_INTX; > + > + /* allocate the mapping details for secondary processes*/ > + *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); > + if (*uio_res == NULL) { > + VMBUS_LOG(ERR, "cannot store uio mmap details"); > + goto error; > + } > + > + strlcpy((*uio_res)->path, devname, PATH_MAX); > + rte_uuid_copy((*uio_res)->id, dev->device_id); > + > + return 0; > + > +error: > + vmbus_uio_free_resource(dev, *uio_res); > + return -1; > +} > + > +static int > +find_max_end_va(const struct rte_memseg_list *msl, void *arg) { > + size_t sz = msl->memseg_arr.len * msl->page_sz; > + void *end_va = RTE_PTR_ADD(msl->base_va, sz); > + void **max_va = arg; > + > + if (*max_va < end_va) > + *max_va = end_va; > + return 0; > +} > + > +/* > + * TODO: this should be part of memseg api. > + * code is duplicated from PCI. > + */ > +static void * > +vmbus_find_max_end_va(void) > +{ > + void *va = NULL; > + > + rte_memseg_list_walk(find_max_end_va, &va); > + return va; > +} > + > +int > +vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int idx, > + struct mapped_vmbus_resource *uio_res, > + int flags) { > + size_t size = dev->resource[idx].len; > + struct vmbus_map *maps = uio_res->maps; > + void *mapaddr; > + off_t offset; > + int fd; > + > + /* devname for mmap */ > + fd = open(uio_res->path, O_RDWR); > + if (fd < 0) { > + VMBUS_LOG(ERR, "Cannot open %s: %s", > + uio_res->path, strerror(errno)); > + return -1; > + } > + > + /* try mapping somewhere close to the end of hugepages */ > + if (vmbus_map_addr == NULL) > + vmbus_map_addr = vmbus_find_max_end_va(); > + > + /* offset is special in uio it indicates which resource */ > + offset = idx * rte_mem_page_size(); > + > + mapaddr = vmbus_map_resource(vmbus_map_addr, fd, offset, size, flags); > + close(fd); > + > + if (mapaddr == MAP_FAILED) > + return -1; > + > + dev->resource[idx].addr = mapaddr; > + vmbus_map_addr = RTE_PTR_ADD(mapaddr, size); > + > + /* Record result of successful mapping for use by secondary */ > + maps[idx].addr = mapaddr; > + maps[idx].size = size; > + > + return 0; > +} > + > +static int vmbus_uio_map_primary(struct vmbus_channel *chan, > + void **ring_buf, uint32_t *ring_size) { > + struct mapped_vmbus_resource *uio_res; > + > + uio_res = vmbus_uio_find_resource(chan->device); > + if (!uio_res) { > + VMBUS_LOG(ERR, "can not find resources!"); > + return -ENOMEM; > + } > + > + if (uio_res->nb_maps < VMBUS_MAX_RESOURCE) { > + VMBUS_LOG(ERR, "VMBUS: only %u resources found!", > + uio_res->nb_maps); > + return -EINVAL; > + } > + > + *ring_size = uio_res->maps[HV_TXRX_RING_MAP].size / 2; > + *ring_buf = uio_res->maps[HV_TXRX_RING_MAP].addr; > + return 0; > +} > + > +static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev, > + const struct vmbus_channel *chan, > + void **ring_buf, uint32_t *ring_size) { > + char ring_path[PATH_MAX]; > + size_t file_size; > + struct stat sb; > + void *mapaddr; > + int fd; > + > + snprintf(ring_path, sizeof(ring_path), > + "%s/%s/channels/%u/ring", > + SYSFS_VMBUS_DEVICES, dev->device.name, > + chan->relid); > + > + fd = open(ring_path, O_RDWR); > + if (fd < 0) { > + VMBUS_LOG(ERR, "Cannot open %s: %s", > + ring_path, strerror(errno)); > + return -errno; > + } > + > + if (fstat(fd, &sb) < 0) { > + VMBUS_LOG(ERR, "Cannot state %s: %s", > + ring_path, strerror(errno)); > + close(fd); > + return -errno; > + } > + file_size = sb.st_size; > + > + if (file_size == 0 || (file_size & (rte_mem_page_size() - 1))) { > + VMBUS_LOG(ERR, "incorrect size %s: %zu", > + ring_path, file_size); > + > + close(fd); > + return -EINVAL; > + } > + > + mapaddr = vmbus_map_resource(vmbus_map_addr, fd, > + 0, file_size, 0); > + close(fd); > + > + if (mapaddr == MAP_FAILED) > + return -EIO; > + > + *ring_size = file_size / 2; > + *ring_buf = mapaddr; > + > + vmbus_map_addr = RTE_PTR_ADD(mapaddr, file_size); > + return 0; > +} > + > +int > +vmbus_uio_map_secondary_subchan(const struct rte_vmbus_device *dev, > + const struct vmbus_channel *chan) { > + const struct vmbus_br *br = &chan->txbr; > + char ring_path[PATH_MAX]; > + void *mapaddr, *ring_buf; > + uint32_t ring_size; > + int fd; > + > + snprintf(ring_path, sizeof(ring_path), > + "%s/%s/channels/%u/ring", > + SYSFS_VMBUS_DEVICES, dev->device.name, > + chan->relid); > + > + ring_buf = br->vbr; > + ring_size = br->dsize + sizeof(struct vmbus_bufring); > + VMBUS_LOG(INFO, "secondary ring_buf %p size %u", > + ring_buf, ring_size); > + > + fd = open(ring_path, O_RDWR); > + if (fd < 0) { > + VMBUS_LOG(ERR, "Cannot open %s: %s", > + ring_path, strerror(errno)); > + return -errno; > + } > + > + mapaddr = vmbus_map_resource(ring_buf, fd, 0, 2 * ring_size, 0); > + close(fd); > + > + if (mapaddr == ring_buf) > + return 0; > + > + if (mapaddr == MAP_FAILED) > + VMBUS_LOG(ERR, > + "mmap subchan %u in secondary failed", chan->relid); > + else { > + VMBUS_LOG(ERR, > + "mmap subchan %u in secondary address mismatch", > + chan->relid); > + vmbus_unmap_resource(mapaddr, 2 * ring_size); > + } > + return -1; > +} > + > +int vmbus_uio_map_rings(struct vmbus_channel *chan) { > + const struct rte_vmbus_device *dev = chan->device; > + uint32_t ring_size; > + void *ring_buf; > + int ret; > + > + /* Primary channel */ > + if (chan->subchannel_id == 0) > + ret = vmbus_uio_map_primary(chan, &ring_buf, &ring_size); > + else > + ret = vmbus_uio_map_subchan(dev, chan, &ring_buf, > + &ring_size); > + > + if (ret) > + return ret; > + > + vmbus_br_setup(&chan->txbr, ring_buf, ring_size); > + vmbus_br_setup(&chan->rxbr, (char *)ring_buf + ring_size, ring_size); > + return 0; > +} > + > +static int vmbus_uio_sysfs_read(const char *dir, const char *name, > + unsigned long *val, unsigned long > +max_range) { > + char path[PATH_MAX]; > + FILE *f; > + int ret; > + > + snprintf(path, sizeof(path), "%s/%s", dir, name); > + f = fopen(path, "r"); > + if (!f) { > + VMBUS_LOG(ERR, "can't open %s:%s", > + path, strerror(errno)); > + return -errno; > + } > + > + if (fscanf(f, "%lu", val) != 1) > + ret = -EIO; > + else if (*val > max_range) > + ret = -ERANGE; > + else > + ret = 0; > + fclose(f); > + > + return ret; > +} > + > +static bool vmbus_uio_ring_present(const struct rte_vmbus_device *dev, > + uint32_t relid) { > + char ring_path[PATH_MAX]; > + > + /* Check if kernel has subchannel sysfs files */ > + snprintf(ring_path, sizeof(ring_path), > + "%s/%s/channels/%u/ring", > + SYSFS_VMBUS_DEVICES, dev->device.name, relid); > + > + return access(ring_path, R_OK|W_OK) == 0; } > + > +bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, > + const struct vmbus_channel *chan) { > + return vmbus_uio_ring_present(dev, chan->relid); } > + > +static bool vmbus_isnew_subchannel(struct vmbus_channel *primary, > + unsigned long id) { > + const struct vmbus_channel *c; > + > + STAILQ_FOREACH(c, &primary->subchannel_list, next) { > + if (c->relid == id) > + return false; > + } > + return true; > +} > + > +int vmbus_uio_get_subchan(struct vmbus_channel *primary, > + struct vmbus_channel **subchan) { > + const struct rte_vmbus_device *dev = primary->device; > + char chan_path[PATH_MAX], subchan_path[PATH_MAX]; > + struct dirent *ent; > + DIR *chan_dir; > + int err; > + > + snprintf(chan_path, sizeof(chan_path), > + "%s/%s/channels", > + SYSFS_VMBUS_DEVICES, dev->device.name); > + > + chan_dir = opendir(chan_path); > + if (!chan_dir) { > + VMBUS_LOG(ERR, "cannot open %s: %s", > + chan_path, strerror(errno)); > + return -errno; > + } > + > + while ((ent = readdir(chan_dir))) { > + unsigned long relid, subid, monid; > + char *endp; > + > + if (ent->d_name[0] == '.') > + continue; > + > + errno = 0; > + relid = strtoul(ent->d_name, &endp, 0); > + if (*endp || errno != 0 || relid > UINT16_MAX) { > + VMBUS_LOG(NOTICE, "not a valid channel relid: %s", > + ent->d_name); > + continue; > + } > + > + if (!vmbus_isnew_subchannel(primary, relid)) { > + VMBUS_LOG(DEBUG, "skip already found channel: %lu", > + relid); > + continue; > + } > + > + if (!vmbus_uio_ring_present(dev, relid)) { > + VMBUS_LOG(DEBUG, "ring mmap not found (yet) for: %lu", > + relid); > + continue; > + } > + > + snprintf(subchan_path, sizeof(subchan_path), "%s/%lu", > + chan_path, relid); > + err = vmbus_uio_sysfs_read(subchan_path, "subchannel_id", > + &subid, UINT16_MAX); > + if (err) { > + VMBUS_LOG(NOTICE, "no subchannel_id in %s:%s", > + subchan_path, strerror(-err)); > + goto fail; > + } > + > + if (subid == 0) > + continue; /* skip primary channel */ > + > + err = vmbus_uio_sysfs_read(subchan_path, "monitor_id", > + &monid, UINT8_MAX); > + if (err) { > + VMBUS_LOG(NOTICE, "no monitor_id in %s:%s", > + subchan_path, strerror(-err)); > + goto fail; > + } > + > + err = vmbus_chan_create(dev, relid, subid, monid, subchan); > + if (err) { > + VMBUS_LOG(ERR, "subchannel setup failed"); > + goto fail; > + } > + break; > + } > + closedir(chan_dir); > + > + return (ent == NULL) ? -ENOENT : 0; > +fail: > + closedir(chan_dir); > + return err; > +} > -- > 2.30.2
> Subject: [PATCH v2 05/11] bus/vmbus: open subchannels > > In FreeBSD, unlike Linux there is no sub-channel open callback that could be > called by HV_UIO driver, upon their grant by the hypervisor. > Thus, the PMD makes an IOCTL to the HV_UIO to open the granted sub-channels > > v2 - Added comment in linux/vmbus_uio.c > > Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> > Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> > Signed-off-by: Anand Thulasiram <avelu@juniper.net> Reviewed-by: Long Li <longli@microsoft.com> > --- > drivers/bus/vmbus/freebsd/vmbus_uio.c | 31 +++++++++++++++++++++++++++ > drivers/bus/vmbus/linux/vmbus_uio.c | 12 +++++++++++ > drivers/bus/vmbus/private.h | 1 + > drivers/bus/vmbus/rte_bus_vmbus.h | 10 +++++++++ > drivers/bus/vmbus/version.map | 1 + > drivers/bus/vmbus/vmbus_channel.c | 5 +++++ > 6 files changed, 60 insertions(+) > > diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c > b/drivers/bus/vmbus/freebsd/vmbus_uio.c > index fdd37dac3a..022ac85302 100644 > --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c > +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c > @@ -12,6 +12,7 @@ > #include <sys/mman.h> > #include <sys/types.h> > #include <sys/sysctl.h> > +#include <sys/ioctl.h> > > #include <rte_log.h> > #include <rte_bus.h> > @@ -26,6 +27,9 @@ > /** Pathname of VMBUS devices directory. */ #define SYSFS_VMBUS_DEVICES > "/sys/bus/vmbus/devices" > > +/* ioctl */ > +#define HVIOOPENSUBCHAN _IOW('h', 14, uint32_t) > + > const char *driver_name = "hv_uio"; > static void *vmbus_map_addr; > > @@ -515,3 +519,30 @@ int vmbus_uio_get_subchan(struct vmbus_channel > *primary, > closedir(chan_dir); > return err; > } > + > +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, uint32_t > +subchan) { > + struct mapped_vmbus_resource *uio_res; > + int fd, err = 0; > + > + uio_res = vmbus_uio_find_resource(dev); > + if (!uio_res) { > + VMBUS_LOG(ERR, "cannot find uio resource"); > + return -EINVAL; > + } > + > + fd = open(uio_res->path, O_RDWR); > + if (fd < 0) { > + VMBUS_LOG(ERR, "Cannot open %s: %s", > + uio_res->path, strerror(errno)); > + return -1; > + } > + > + if (ioctl(fd, HVIOOPENSUBCHAN, &subchan)) { > + VMBUS_LOG(ERR, "open subchan ioctl failed %s: %s", > + uio_res->path, strerror(errno)); > + err = -1; > + } > + close(fd); > + return err; > +} > diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c > b/drivers/bus/vmbus/linux/vmbus_uio.c > index b52ca5bf1d..29e889347a 100644 > --- a/drivers/bus/vmbus/linux/vmbus_uio.c > +++ b/drivers/bus/vmbus/linux/vmbus_uio.c > @@ -451,3 +451,15 @@ int vmbus_uio_get_subchan(struct vmbus_channel > *primary, > closedir(chan_dir); > return err; > } > + > +/* > + * This is a stub function and it should always succeed. > + * The Linux UIO kernel driver opens the subchannels implicitly. > + */ > +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, > + uint32_t subchan) > +{ > + RTE_SET_USED(dev); > + RTE_SET_USED(subchan); > + return 0; > +} > diff --git a/drivers/bus/vmbus/private.h b/drivers/bus/vmbus/private.h index > 528d60a42f..968f0b6f23 100644 > --- a/drivers/bus/vmbus/private.h > +++ b/drivers/bus/vmbus/private.h > @@ -107,6 +107,7 @@ int vmbus_uio_get_subchan(struct vmbus_channel > *primary, int vmbus_uio_map_rings(struct vmbus_channel *chan); int > vmbus_uio_map_secondary_subchan(const struct rte_vmbus_device *dev, > const struct vmbus_channel *chan); > +int vmbus_uio_subchan_open(struct rte_vmbus_device *device, uint32_t > +subchan); > > void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen); > > diff --git a/drivers/bus/vmbus/rte_bus_vmbus.h > b/drivers/bus/vmbus/rte_bus_vmbus.h > index 6bcff66468..60c5812706 100644 > --- a/drivers/bus/vmbus/rte_bus_vmbus.h > +++ b/drivers/bus/vmbus/rte_bus_vmbus.h > @@ -404,6 +404,16 @@ void rte_vmbus_chan_dump(FILE *f, const struct > vmbus_channel *chan); > */ > void rte_vmbus_unregister(struct rte_vmbus_driver *driver); > > +/** > + * Perform IOCTL to VMBUS device > + * > + * @param device > + * A pointer to a rte_vmbus_device structure > + * @param subchan > + * Count of subchannels to open > + */ > +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan); > + > /** Helper for VMBUS device registration from driver instance */ > #define RTE_PMD_REGISTER_VMBUS(nm, vmbus_drv) \ > RTE_INIT(vmbusinitfn_ ##nm) \ > diff --git a/drivers/bus/vmbus/version.map b/drivers/bus/vmbus/version.map > index 3cadec7fae..3509d4fc14 100644 > --- a/drivers/bus/vmbus/version.map > +++ b/drivers/bus/vmbus/version.map > @@ -23,6 +23,7 @@ DPDK_22 { > rte_vmbus_subchan_open; > rte_vmbus_unmap_device; > rte_vmbus_unregister; > + rte_vmbus_ioctl; > > local: *; > }; > diff --git a/drivers/bus/vmbus/vmbus_channel.c > b/drivers/bus/vmbus/vmbus_channel.c > index f67f1c438a..f53a1b6511 100644 > --- a/drivers/bus/vmbus/vmbus_channel.c > +++ b/drivers/bus/vmbus/vmbus_channel.c > @@ -367,6 +367,11 @@ int rte_vmbus_max_channels(const struct > rte_vmbus_device *device) > return 1; > } > > +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan) > +{ > + return vmbus_uio_subchan_open(device, subchan); } > + > /* Setup secondary channel */ > int rte_vmbus_subchan_open(struct vmbus_channel *primary, > struct vmbus_channel **new_chan) > -- > 2.30.1
> Subject: [PATCH v2 11/11] net/netvsc: add meson support for FreeBSD > > add meson support for FreeBSD > > v2 - moved include statement to after OS detection > - updated ./MAINTAINERS file > > Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> > Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> > Signed-off-by: Anand Thulasiram <avelu@juniper.net> Reviewed-by: Long Li <longli@microsoft.com> > --- > MAINTAINERS | 2 ++ > drivers/net/netvsc/freebsd/meson.build | 6 ++++++ > drivers/net/netvsc/linux/meson.build | 6 ++++++ > drivers/net/netvsc/meson.build | 4 ++++ > 4 files changed, 18 insertions(+) > create mode 100644 drivers/net/netvsc/freebsd/meson.build > create mode 100644 drivers/net/netvsc/linux/meson.build > > diff --git a/MAINTAINERS b/MAINTAINERS > index 278e5b3226..930d789945 100644 > --- a/MAINTAINERS > +++ b/MAINTAINERS > @@ -552,6 +552,7 @@ F: drivers/bus/vdev/ VMBUS bus driver > M: Stephen Hemminger <sthemmin@microsoft.com> > M: Long Li <longli@microsoft.com> > +M: Srikanth Kaka <srikanth.k@oneconvergence.com> > F: drivers/bus/vmbus/ > > > @@ -824,6 +825,7 @@ F: doc/guides/nics/vdev_netvsc.rst Microsoft Hyper-V > netvsc > M: Stephen Hemminger <sthemmin@microsoft.com> > M: Long Li <longli@microsoft.com> > +M: Srikanth Kaka <srikanth.k@oneconvergence.com> > F: drivers/net/netvsc/ > F: doc/guides/nics/netvsc.rst > F: doc/guides/nics/features/netvsc.ini > diff --git a/drivers/net/netvsc/freebsd/meson.build > b/drivers/net/netvsc/freebsd/meson.build > new file mode 100644 > index 0000000000..78f824f701 > --- /dev/null > +++ b/drivers/net/netvsc/freebsd/meson.build > @@ -0,0 +1,6 @@ > +# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Microsoft > +Corporation > + > +sources += files( > + 'hn_os.c', > +) > diff --git a/drivers/net/netvsc/linux/meson.build > b/drivers/net/netvsc/linux/meson.build > new file mode 100644 > index 0000000000..78f824f701 > --- /dev/null > +++ b/drivers/net/netvsc/linux/meson.build > @@ -0,0 +1,6 @@ > +# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Microsoft > +Corporation > + > +sources += files( > + 'hn_os.c', > +) > diff --git a/drivers/net/netvsc/meson.build b/drivers/net/netvsc/meson.build > index f74b941f65..6c9f5cfc56 100644 > --- a/drivers/net/netvsc/meson.build > +++ b/drivers/net/netvsc/meson.build > @@ -7,6 +7,8 @@ if is_windows > subdir_done() > endif > > +includes += include_directories(exec_env) > + > build = dpdk_conf.has('RTE_BUS_VMBUS') > reason = 'missing dependency, DPDK VMBus driver' > sources = files( > @@ -18,3 +20,5 @@ sources = files( > ) > > deps += ['bus_vmbus' ] > + > +subdir(exec_env) > -- > 2.30.1
> Subject: [PATCH 08/11] net/netvsc: moving event monitoring support > > [You don't often get email from srikanth.k@oneconvergence.com. Learn why > this is important at http://aka.ms/LearnAboutSenderIdentification.] > > Event monitoring is not yet supported on FreeBSD, hence moving it the OS > specific files > > Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> > Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> > Signed-off-by: Anand Thulasiram <avelu@juniper.net> Reviewed-by: Long Li <longli@microsoft.com> > --- > drivers/net/netvsc/freebsd/hn_os.c | 14 ++++++++++++++ > drivers/net/netvsc/hn_ethdev.c | 7 +++---- > drivers/net/netvsc/hn_os.h | 6 ++++++ > drivers/net/netvsc/linux/hn_os.c | 19 +++++++++++++++++++ > 4 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 > drivers/net/netvsc/freebsd/hn_os.c > create mode 100644 drivers/net/netvsc/hn_os.h create mode 100644 > drivers/net/netvsc/linux/hn_os.c > > diff --git a/drivers/net/netvsc/freebsd/hn_os.c > b/drivers/net/netvsc/freebsd/hn_os.c > new file mode 100644 > index 0000000000..3bd67e06c8 > --- /dev/null > +++ b/drivers/net/netvsc/freebsd/hn_os.c > @@ -0,0 +1,14 @@ > +/* SPDX-License-Identifier: BSD-3-Clause > + * Copyright(c) 2016-2021 Microsoft Corporation */ > + > +#include <stdio.h> > + > +#include "hn_logs.h" > +#include "hn_os.h" > + > +int eth_hn_os_dev_event(void) > +{ > + PMD_DRV_LOG(DEBUG, "rte_dev_event_monitor_start not supported on > FreeBSD"); > + return 0; > +} > diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c > index 6f9053c4d5..61d7d3daeb 100644 > --- a/drivers/net/netvsc/hn_ethdev.c > +++ b/drivers/net/netvsc/hn_ethdev.c > @@ -39,6 +39,7 @@ > #include "hn_rndis.h" > #include "hn_nvs.h" > #include "ndis.h" > +#include "hn_os.h" > > #define HN_TX_OFFLOAD_CAPS (DEV_TX_OFFLOAD_IPV4_CKSUM | \ > DEV_TX_OFFLOAD_TCP_CKSUM | \ @@ -1244,11 +1245,9 @@ > static int eth_hn_probe(struct rte_vmbus_driver *drv __rte_unused, > > PMD_INIT_FUNC_TRACE(); > > - ret = rte_dev_event_monitor_start(); > - if (ret) { > - PMD_DRV_LOG(ERR, "Failed to start device event monitoring"); > + ret = eth_hn_os_dev_event(); > + if (ret) > return ret; > - } > > eth_dev = eth_dev_vmbus_allocate(dev, sizeof(struct hn_data)); > if (!eth_dev) > diff --git a/drivers/net/netvsc/hn_os.h b/drivers/net/netvsc/hn_os.h new file > mode 100644 index 0000000000..618c53cdcd > --- /dev/null > +++ b/drivers/net/netvsc/hn_os.h > @@ -0,0 +1,6 @@ > +/* SPDX-License-Identifier: BSD-3-Clause > + * Copyright (c) 2009-2021 Microsoft Corp. > + * All rights reserved. > + */ > + > +int eth_hn_os_dev_event(void); > diff --git a/drivers/net/netvsc/linux/hn_os.c b/drivers/net/netvsc/linux/hn_os.c > new file mode 100644 > index 0000000000..862dc190c1 > --- /dev/null > +++ b/drivers/net/netvsc/linux/hn_os.c > @@ -0,0 +1,19 @@ > +/* SPDX-License-Identifier: BSD-3-Clause > + * Copyright(c) 2016-2021 Microsoft Corporation */ > + > +#include <rte_ethdev.h> > + > +#include "hn_logs.h" > +#include "hn_os.h" > + > +int eth_hn_os_dev_event(void) > +{ > + int ret; > + > + ret = rte_dev_event_monitor_start(); > + if (ret) > + PMD_DRV_LOG(ERR, "Failed to start device event > + monitoring"); > + > + return ret; > +} > -- > 2.30.2
> Subject: [PATCH 09/11] net/netvsc: moving hotplug retry to OS dir > > [You don't often get email from srikanth.k@oneconvergence.com. Learn why > this is important at http://aka.ms/LearnAboutSenderIdentification.] > > Moved netvsc_hotplug_retry to respective OS dir as it contains OS dependent > code. For Linux, it is copied as is and for FreeBSD it is not supported yet. > > Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> > Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> > Signed-off-by: Anand Thulasiram <avelu@juniper.net> Reviewed-by: Long Li <longli@microsoft.com> > --- > drivers/net/netvsc/freebsd/hn_os.c | 8 +++ > drivers/net/netvsc/hn_ethdev.c | 84 --------------------------- > drivers/net/netvsc/hn_os.h | 2 + > drivers/net/netvsc/linux/hn_os.c | 92 ++++++++++++++++++++++++++++++ > 4 files changed, 102 insertions(+), 84 deletions(-) > > diff --git a/drivers/net/netvsc/freebsd/hn_os.c > b/drivers/net/netvsc/freebsd/hn_os.c > index 3bd67e06c8..2ba4c32a76 100644 > --- a/drivers/net/netvsc/freebsd/hn_os.c > +++ b/drivers/net/netvsc/freebsd/hn_os.c > @@ -4,6 +4,8 @@ > > #include <stdio.h> > > +#include <rte_common.h> > + > #include "hn_logs.h" > #include "hn_os.h" > > @@ -12,3 +14,9 @@ int eth_hn_os_dev_event(void) > PMD_DRV_LOG(DEBUG, "rte_dev_event_monitor_start not supported on > FreeBSD"); > return 0; > } > + > +void netvsc_hotplug_retry(void *args) > +{ > + RTE_SET_USED(args); > + return; > +} > diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c > index 61d7d3daeb..c299b98a6c 100644 > --- a/drivers/net/netvsc/hn_ethdev.c > +++ b/drivers/net/netvsc/hn_ethdev.c > @@ -57,9 +57,6 @@ > #define NETVSC_ARG_TXBREAK "tx_copybreak" > #define NETVSC_ARG_RX_EXTMBUF_ENABLE "rx_extmbuf_enable" > > -/* The max number of retry when hot adding a VF device */ -#define > NETVSC_MAX_HOTADD_RETRY 10 > - > struct hn_xstats_name_off { > char name[RTE_ETH_XSTATS_NAME_SIZE]; > unsigned int offset; > @@ -556,87 +553,6 @@ static int hn_subchan_configure(struct hn_data *hv, > return err; > } > > -static void netvsc_hotplug_retry(void *args) -{ > - int ret; > - struct hn_data *hv = args; > - struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id]; > - struct rte_devargs *d = &hv->devargs; > - char buf[256]; > - > - DIR *di; > - struct dirent *dir; > - struct ifreq req; > - struct rte_ether_addr eth_addr; > - int s; > - > - PMD_DRV_LOG(DEBUG, "%s: retry count %d", > - __func__, hv->eal_hot_plug_retry); > - > - if (hv->eal_hot_plug_retry++ > NETVSC_MAX_HOTADD_RETRY) > - return; > - > - snprintf(buf, sizeof(buf), "/sys/bus/pci/devices/%s/net", d->name); > - di = opendir(buf); > - if (!di) { > - PMD_DRV_LOG(DEBUG, "%s: can't open directory %s, " > - "retrying in 1 second", __func__, buf); > - goto retry; > - } > - > - while ((dir = readdir(di))) { > - /* Skip . and .. directories */ > - if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) > - continue; > - > - /* trying to get mac address if this is a network device*/ > - s = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); > - if (s == -1) { > - PMD_DRV_LOG(ERR, "Failed to create socket errno %d", > - errno); > - break; > - } > - strlcpy(req.ifr_name, dir->d_name, sizeof(req.ifr_name)); > - ret = ioctl(s, SIOCGIFHWADDR, &req); > - close(s); > - if (ret == -1) { > - PMD_DRV_LOG(ERR, > - "Failed to send SIOCGIFHWADDR for device %s", > - dir->d_name); > - break; > - } > - if (req.ifr_hwaddr.sa_family != ARPHRD_ETHER) { > - closedir(di); > - return; > - } > - memcpy(eth_addr.addr_bytes, req.ifr_hwaddr.sa_data, > - RTE_DIM(eth_addr.addr_bytes)); > - > - if (rte_is_same_ether_addr(ð_addr, dev->data->mac_addrs)) { > - PMD_DRV_LOG(NOTICE, > - "Found matching MAC address, adding device %s network > name %s", > - d->name, dir->d_name); > - ret = rte_eal_hotplug_add(d->bus->name, d->name, > - d->args); > - if (ret) { > - PMD_DRV_LOG(ERR, > - "Failed to add PCI device %s", > - d->name); > - break; > - } > - } > - /* When the code reaches here, we either have already added > - * the device, or its MAC address did not match. > - */ > - closedir(di); > - return; > - } > - closedir(di); > -retry: > - /* The device is still being initialized, retry after 1 second */ > - rte_eal_alarm_set(1000000, netvsc_hotplug_retry, hv); > -} > - > static void > netvsc_hotadd_callback(const char *device_name, enum rte_dev_event_type > type, > void *arg) > diff --git a/drivers/net/netvsc/hn_os.h b/drivers/net/netvsc/hn_os.h index > 618c53cdcd..1fb7292b17 100644 > --- a/drivers/net/netvsc/hn_os.h > +++ b/drivers/net/netvsc/hn_os.h > @@ -4,3 +4,5 @@ > */ > > int eth_hn_os_dev_event(void); > + > +void netvsc_hotplug_retry(void *args); > diff --git a/drivers/net/netvsc/linux/hn_os.c b/drivers/net/netvsc/linux/hn_os.c > index 862dc190c1..9c2f4cd7a8 100644 > --- a/drivers/net/netvsc/linux/hn_os.c > +++ b/drivers/net/netvsc/linux/hn_os.c > @@ -2,11 +2,22 @@ > * Copyright(c) 2016-2021 Microsoft Corporation > */ > > +#include <unistd.h> > +#include <dirent.h> > +#include <net/if.h> > +#include <net/if_arp.h> > +#include <sys/ioctl.h> > + > #include <rte_ethdev.h> > +#include <rte_alarm.h> > > #include "hn_logs.h" > +#include "hn_var.h" > #include "hn_os.h" > > +/* The max number of retry when hot adding a VF device */ #define > +NETVSC_MAX_HOTADD_RETRY 10 > + > int eth_hn_os_dev_event(void) > { > int ret; > @@ -17,3 +28,84 @@ int eth_hn_os_dev_event(void) > > return ret; > } > + > +void netvsc_hotplug_retry(void *args) > +{ > + int ret; > + struct hn_data *hv = args; > + struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id]; > + struct rte_devargs *d = &hv->devargs; > + char buf[256]; > + > + DIR *di; > + struct dirent *dir; > + struct ifreq req; > + struct rte_ether_addr eth_addr; > + int s; > + > + PMD_DRV_LOG(DEBUG, "%s: retry count %d", > + __func__, hv->eal_hot_plug_retry); > + > + if (hv->eal_hot_plug_retry++ > NETVSC_MAX_HOTADD_RETRY) > + return; > + > + snprintf(buf, sizeof(buf), "/sys/bus/pci/devices/%s/net", d->name); > + di = opendir(buf); > + if (!di) { > + PMD_DRV_LOG(DEBUG, "%s: can't open directory %s, " > + "retrying in 1 second", __func__, buf); > + goto retry; > + } > + > + while ((dir = readdir(di))) { > + /* Skip . and .. directories */ > + if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) > + continue; > + > + /* trying to get mac address if this is a network device*/ > + s = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); > + if (s == -1) { > + PMD_DRV_LOG(ERR, "Failed to create socket errno %d", > + errno); > + break; > + } > + strlcpy(req.ifr_name, dir->d_name, sizeof(req.ifr_name)); > + ret = ioctl(s, SIOCGIFHWADDR, &req); > + close(s); > + if (ret == -1) { > + PMD_DRV_LOG(ERR, > + "Failed to send SIOCGIFHWADDR for device %s", > + dir->d_name); > + break; > + } > + if (req.ifr_hwaddr.sa_family != ARPHRD_ETHER) { > + closedir(di); > + return; > + } > + memcpy(eth_addr.addr_bytes, req.ifr_hwaddr.sa_data, > + RTE_DIM(eth_addr.addr_bytes)); > + > + if (rte_is_same_ether_addr(ð_addr, dev->data->mac_addrs)) { > + PMD_DRV_LOG(NOTICE, > + "Found matching MAC address, adding device %s network > name %s", > + d->name, dir->d_name); > + ret = rte_eal_hotplug_add(d->bus->name, d->name, > + d->args); > + if (ret) { > + PMD_DRV_LOG(ERR, > + "Failed to add PCI device %s", > + d->name); > + break; > + } > + } > + /* When the code reaches here, we either have already added > + * the device, or its MAC address did not match. > + */ > + closedir(di); > + return; > + } > + closedir(di); > +retry: > + /* The device is still being initialized, retry after 1 second */ > + rte_eal_alarm_set(1000000, netvsc_hotplug_retry, hv); } > -- > 2.30.2
> Subject: [PATCH 10/11] bus/vmbus: add meson support for FreeBSD OS > > [You don't often get email from srikanth.k@oneconvergence.com. Learn why > this is important at http://aka.ms/LearnAboutSenderIdentification.] > > add meson support for FreeBSD OS > > Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> > Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> > Signed-off-by: Anand Thulasiram <avelu@juniper.net> Reviewed-by: Long Li <longli@microsoft.com> > --- > drivers/bus/vmbus/meson.build | 6 +++++- > 1 file changed, 5 insertions(+), 1 deletion(-) > > diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build > index 3892cbf67f..8fbe144fcd 100644 > --- a/drivers/bus/vmbus/meson.build > +++ b/drivers/bus/vmbus/meson.build > @@ -20,7 +20,11 @@ if is_linux > sources += files('linux/vmbus_bus.c', > 'linux/vmbus_uio.c') > includes += include_directories('linux') > +elif is_freebsd > + sources += files('freebsd/vmbus_bus.c', > + 'freebsd/vmbus_uio.c') > + includes += include_directories('freebsd') > else > build = false > - reason = 'only supported on Linux' > + reason = 'only supported on Linux & FreeBSD' > endif > -- > 2.30.2
06/11/2021 21:49, Long Li:
> > Subject: [PATCH v2 05/11] bus/vmbus: open subchannels
> >
> > In FreeBSD, unlike Linux there is no sub-channel open callback that could be
> > called by HV_UIO driver, upon their grant by the hypervisor.
> > Thus, the PMD makes an IOCTL to the HV_UIO to open the granted sub-channels
> >
> > v2 - Added comment in linux/vmbus_uio.c
> >
> > Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com>
> > Signed-off-by: Vag Singh <vag.singh@oneconvergence.com>
> > Signed-off-by: Anand Thulasiram <avelu@juniper.net>
>
> Reviewed-by: Long Li <longli@microsoft.com>
This series is not handled because it looks incomplete.
Please resend all patches as v3 as a reply to the cover letter of v1.
Then we should have a complete CI run on the series, hopefully green.
This patch series adds support to VMBUS & NetVSC PMDs run on FreeBSD Changelog: v3: - split the patches into further logical parts - updated docs v2: - replaced strncpy with memcpy - replaced malloc.h with stdlib.h - added comment in linux/vmbus_uio.c v1: Intial release Srikanth Kaka (15): bus/vmbus: scan and get the network device bus/vmbus: handle mapping of device resources bus/vmbus: get device resource values using sysctl bus/vmbus: add resource by index net/netvsc: make event monitor OS dependent bus/vmbus: add ring mapping APIs bus/vmbus: add stub for subchannel support API bus/vmbus: open subchannels net/netvsc: make IOCTL call to open subchannels bus/vmbus: get subchannel info net/netvsc: moving hotplug retry to OS dir bus/vmbus: add meson support for FreeBSD net/netvsc: add meson support for FreeBSD bus/vmbus: add APIs to mask/unmask IRQs bus/vmbus: update MAINTAINERS and docs MAINTAINERS | 2 + doc/guides/nics/netvsc.rst | 11 + drivers/bus/vmbus/freebsd/vmbus_bus.c | 282 ++++++++++++++++ drivers/bus/vmbus/freebsd/vmbus_uio.c | 449 +++++++++++++++++++++++++ drivers/bus/vmbus/linux/vmbus_uio.c | 12 + drivers/bus/vmbus/meson.build | 6 +- drivers/bus/vmbus/private.h | 1 + drivers/bus/vmbus/rte_bus_vmbus.h | 10 + drivers/bus/vmbus/version.map | 1 + drivers/bus/vmbus/vmbus_channel.c | 5 + drivers/net/netvsc/freebsd/hn_os.c | 22 ++ drivers/net/netvsc/freebsd/meson.build | 6 + drivers/net/netvsc/hn_ethdev.c | 95 +----- drivers/net/netvsc/hn_os.h | 8 + drivers/net/netvsc/linux/hn_os.c | 111 ++++++ drivers/net/netvsc/linux/meson.build | 6 + drivers/net/netvsc/meson.build | 3 + 17 files changed, 941 insertions(+), 89 deletions(-) create mode 100644 drivers/bus/vmbus/freebsd/vmbus_bus.c create mode 100644 drivers/bus/vmbus/freebsd/vmbus_uio.c create mode 100644 drivers/net/netvsc/freebsd/hn_os.c create mode 100644 drivers/net/netvsc/freebsd/meson.build create mode 100644 drivers/net/netvsc/hn_os.h create mode 100644 drivers/net/netvsc/linux/hn_os.c create mode 100644 drivers/net/netvsc/linux/meson.build -- 2.30.2
Using sysctl, all the devices on the VMBUS are identified by the PMD. On finding the Network device's device id, it is added to VMBUS dev list. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_bus.c | 249 ++++++++++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 drivers/bus/vmbus/freebsd/vmbus_bus.c diff --git a/drivers/bus/vmbus/freebsd/vmbus_bus.c b/drivers/bus/vmbus/freebsd/vmbus_bus.c new file mode 100644 index 0000000000..c8583aba37 --- /dev/null +++ b/drivers/bus/vmbus/freebsd/vmbus_bus.c @@ -0,0 +1,249 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <rte_eal.h> +#include <rte_bus_vmbus.h> + +#include "private.h" + +#include <sys/bus.h> +#include <sys/types.h> +#include <sys/sysctl.h> + +/* + * GUID associated with network devices + * {f8615163-df3e-46c5-913f-f2d2f965ed0e} + */ +static const rte_uuid_t vmbus_nic_uuid = { + 0xf8, 0x61, 0x51, 0x63, + 0xdf, 0x3e, + 0x46, 0xc5, + 0x91, 0x3f, + 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0xe +}; + +extern struct rte_vmbus_bus rte_vmbus_bus; + +/* Parse UUID. Caller must pass NULL terminated string */ +static int +parse_sysfs_uuid(const char *filename, rte_uuid_t uu) +{ + char in[BUFSIZ]; + + memcpy(in, filename, BUFSIZ); + if (rte_uuid_parse(in, uu) < 0) { + VMBUS_LOG(ERR, "%s not a valid UUID", in); + return -1; + } + + return 0; +} + +/* Scan one vmbus entry, and fill the devices list from it. */ +static int +vmbus_scan_one(const char *name, unsigned int unit_num) +{ + struct rte_vmbus_device *dev, *dev2; + char sysctlBuffer[PATH_MAX], sysctlVar[PATH_MAX]; + size_t guid_len = 36, len = PATH_MAX; + char classid[guid_len + 1], deviceid[guid_len + 1]; + + dev = calloc(1, sizeof(*dev)); + if (dev == NULL) + return -1; + + /* get class id and device id */ + snprintf(sysctlVar, len, "dev.%s.%u.%%pnpinfo", name, unit_num); + if (sysctlbyname(sysctlVar, &sysctlBuffer, &len, NULL, 0) < 0) + goto error; + + /* pnpinfo: classid=f912ad6d-2b17-48ea-bd65-f927a61c7684 + * deviceid=d34b2567-b9b6-42b9-8778-0a4ec0b955bf + */ + if (sysctlBuffer[0] == 'c' && sysctlBuffer[1] == 'l' && + sysctlBuffer[7] == '=') { + memcpy(classid, &sysctlBuffer[8], guid_len); + classid[guid_len] = '\0'; + } + if (parse_sysfs_uuid(classid, dev->class_id) < 0) + goto error; + + /* skip non-network devices */ + if (rte_uuid_compare(dev->class_id, vmbus_nic_uuid) != 0) { + free(dev); + return 0; + } + + if (sysctlBuffer[45] == 'd' && sysctlBuffer[46] == 'e' && + sysctlBuffer[47] == 'v' && sysctlBuffer[53] == '=') { + memcpy(deviceid, &sysctlBuffer[54], guid_len); + deviceid[guid_len] = '\0'; + } + if (parse_sysfs_uuid(deviceid, dev->device_id) < 0) + goto error; + + if (!strcmp(name, "hv_uio")) + dev->uio_num = unit_num; + else + dev->uio_num = -1; + dev->device.bus = &rte_vmbus_bus.bus; + dev->device.numa_node = 0; + dev->device.name = strdup(deviceid); + if (!dev->device.name) + goto error; + + dev->device.devargs = vmbus_devargs_lookup(dev); + + /* device is valid, add in list (sorted) */ + VMBUS_LOG(DEBUG, "Adding vmbus device %s", name); + + TAILQ_FOREACH(dev2, &rte_vmbus_bus.device_list, next) { + int ret; + + ret = rte_uuid_compare(dev->device_id, dev2->device_id); + if (ret > 0) + continue; + + if (ret < 0) { + vmbus_insert_device(dev2, dev); + } else { /* already registered */ + VMBUS_LOG(NOTICE, + "%s already registered", name); + free(dev); + } + return 0; + } + + vmbus_add_device(dev); + return 0; +error: + VMBUS_LOG(DEBUG, "failed"); + + free(dev); + return -1; +} + +/* + * Scan the content of the vmbus, and the devices in the devices list + */ +int +rte_vmbus_scan(void) +{ + struct u_device udev; + struct u_businfo ubus; + int dev_idx, dev_ptr, name2oid[2], oid[CTL_MAXNAME + 12], error; + size_t oidlen, rlen, ub_size; + uintptr_t vmbus_handle = 0; + char *walker, *ep; + char name[16] = "hw.bus.devices"; + char *dd_name, *dd_desc, *dd_drivername, *dd_pnpinfo, *dd_location; + + /* + * devinfo FreeBSD APP logic to fetch all the VMBus devices + * using SYSCTLs + */ + name2oid[0] = 0; + name2oid[1] = 3; + oidlen = sizeof(oid); + error = sysctl(name2oid, 2, oid, &oidlen, name, strlen(name)); + if (error < 0) { + VMBUS_LOG(DEBUG, "can't find hw.bus.devices sysctl node"); + return -ENOENT; + } + oidlen /= sizeof(int); + if (oidlen > CTL_MAXNAME) { + VMBUS_LOG(DEBUG, "hw.bus.devices oid is too large"); + return -EINVAL; + } + + ub_size = sizeof(ubus); + if (sysctlbyname("hw.bus.info", &ubus, &ub_size, NULL, 0) != 0) { + VMBUS_LOG(DEBUG, "sysctlbyname(\"hw.bus.info\", ...) failed"); + return -EINVAL; + } + if ((ub_size != sizeof(ubus)) || + (ubus.ub_version != BUS_USER_VERSION)) { + VMBUS_LOG(DEBUG, + "kernel bus interface version mismatch: kernel %d expected %d", + ubus.ub_version, BUS_USER_VERSION); + return -EINVAL; + } + + oid[oidlen++] = ubus.ub_generation; + dev_ptr = oidlen++; + + /* + * Scan devices. + * + * Stop after a fairly insane number to avoid death in the case + * of kernel corruption. + */ + + for (dev_idx = 0; dev_idx < 10000; dev_idx++) { + /* + * Get the device information. + */ + oid[dev_ptr] = dev_idx; + rlen = sizeof(udev); + error = sysctl(oid, oidlen, &udev, &rlen, NULL, 0); + if (error < 0) { + if (errno == ENOENT) /* end of list */ + break; + if (errno != EINVAL) /* gen count skip, restart */ + VMBUS_LOG(DEBUG, "sysctl hw.bus.devices.%d", + dev_idx); + return errno; + } + if (rlen != sizeof(udev)) { + VMBUS_LOG(DEBUG, + "sysctl returned wrong data %zd bytes instead of %zd", + rlen, sizeof(udev)); + return -EINVAL; + } + + walker = udev.dv_fields; + ep = walker + sizeof(udev.dv_fields); + dd_name = NULL; + dd_desc = NULL; + dd_drivername = NULL; + dd_pnpinfo = NULL; + dd_location = NULL; +#define UNPACK(x) \ + do { \ + x = strdup(walker); \ + if (x == NULL) \ + return -ENOMEM; \ + if (walker + strnlen(walker, ep - walker) >= ep) \ + return -EINVAL; \ + walker += strlen(walker) + 1; \ + } while (0) + + UNPACK(dd_name); + UNPACK(dd_desc); + UNPACK(dd_drivername); + UNPACK(dd_pnpinfo); + UNPACK(dd_location); +#undef UNPACK + if (*dd_drivername && !(strcmp(dd_drivername, "vmbus"))) + vmbus_handle = udev.dv_handle; + + if (vmbus_handle && (vmbus_handle == udev.dv_parent) + && *dd_pnpinfo && *dd_name) { + unsigned int driver_len = 0, unit_num = 0; + char *endptr; + + driver_len = strlen(dd_drivername); + unit_num = strtoull(&dd_name[driver_len], &endptr, 10); + VMBUS_LOG(DEBUG, "Device name:%s, pnpinfo:%s", + dd_name, dd_pnpinfo); + + if (vmbus_scan_one(dd_drivername, unit_num) < 0) + goto error; + } + } + return 0; +error: + return -1; +} -- 2.30.2
All resource values are published by HV_UIO driver as sysctl key value pairs and they are read at a later point of the code flow Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_bus.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_bus.c b/drivers/bus/vmbus/freebsd/vmbus_bus.c index c8583aba37..fa0f4883f7 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_bus.c +++ b/drivers/bus/vmbus/freebsd/vmbus_bus.c @@ -41,6 +41,24 @@ parse_sysfs_uuid(const char *filename, rte_uuid_t uu) return 0; } +/* map the resources of a vmbus device in virtual memory */ +int +rte_vmbus_map_device(struct rte_vmbus_device *dev) +{ + if (dev->uio_num < 0) { + VMBUS_LOG(DEBUG, "Not managed by UIO driver, skipped"); + return 1; + } + + return vmbus_uio_map_resource(dev); +} + +void +rte_vmbus_unmap_device(struct rte_vmbus_device *dev) +{ + vmbus_uio_unmap_resource(dev); +} + /* Scan one vmbus entry, and fill the devices list from it. */ static int vmbus_scan_one(const char *name, unsigned int unit_num) -- 2.30.2
The UIO device's relid, monitor id, etc values are retrieved using following sysctl variables: $ sysctl dev.hv_uio.0 dev.hv_uio.0.send_buf.gpadl: 925241 dev.hv_uio.0.send_buf.size: 16777216 dev.hv_uio.0.recv_buf.gpadl: 925240 dev.hv_uio.0.recv_buf.size: 32505856 dev.hv_uio.0.monitor_page.size: 4096 dev.hv_uio.0.int_page.size: 4096 Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 142 ++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 drivers/bus/vmbus/freebsd/vmbus_uio.c diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c new file mode 100644 index 0000000000..2059ab192b --- /dev/null +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <unistd.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/sysctl.h> +#include <sys/ioctl.h> + +#include <rte_log.h> +#include <rte_bus.h> +#include <rte_malloc.h> +#include <rte_bus_vmbus.h> + +#include "private.h" + +const char *driver_name = "hv_uio"; + +/* Check map names with kernel names */ +static const char *map_names[VMBUS_MAX_RESOURCE] = { + [HV_TXRX_RING_MAP] = "txrx_rings", + [HV_INT_PAGE_MAP] = "int_page", + [HV_MON_PAGE_MAP] = "monitor_page", + [HV_RECV_BUF_MAP] = "recv_buf", + [HV_SEND_BUF_MAP] = "send_buf", +}; + +void +vmbus_uio_free_resource(struct rte_vmbus_device *dev, + struct mapped_vmbus_resource *uio_res) +{ + rte_free(uio_res); + + if (dev->intr_handle.uio_cfg_fd >= 0) { + close(dev->intr_handle.uio_cfg_fd); + dev->intr_handle.uio_cfg_fd = -1; + } + + if (dev->intr_handle.fd >= 0) { + close(dev->intr_handle.fd); + dev->intr_handle.fd = -1; + dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; + } +} + +static int +sysctl_get_vmbus_device_info(struct rte_vmbus_device *dev) +{ + char sysctlBuffer[PATH_MAX]; + char sysctlVar[PATH_MAX]; + size_t len = PATH_MAX, sysctl_len; + unsigned long tmp; + int i; + + snprintf(sysctlBuffer, len, "dev.%s.%d", driver_name, dev->uio_num); + + sysctl_len = sizeof(unsigned long); + /* get relid */ + snprintf(sysctlVar, len, "%s.channel.ch_id", sysctlBuffer); + if (sysctlbyname(sysctlVar, &tmp, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s", sysctlVar); + goto error; + } + dev->relid = tmp; + + /* get monitor id */ + snprintf(sysctlVar, len, "%s.channel.%u.monitor_id", sysctlBuffer, + dev->relid); + if (sysctlbyname(sysctlVar, &tmp, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s", sysctlVar); + goto error; + } + dev->monitor_id = tmp; + + /* Extract resource value */ + for (i = 0; i < VMBUS_MAX_RESOURCE; i++) { + struct rte_mem_resource *res = &dev->resource[i]; + unsigned long size, gpad = 0; + size_t sizelen = sizeof(len); + + snprintf(sysctlVar, sizeof(sysctlVar), "%s.%s.size", + sysctlBuffer, map_names[i]); + if (sysctlbyname(sysctlVar, &size, &sizelen, NULL, 0) < 0) { + VMBUS_LOG(ERR, + "could not read %s", sysctlVar); + goto error; + } + res->len = size; + + if (i == HV_RECV_BUF_MAP || i == HV_SEND_BUF_MAP) { + snprintf(sysctlVar, sizeof(sysctlVar), "%s.%s.gpadl", + sysctlBuffer, map_names[i]); + if (sysctlbyname(sysctlVar, &gpad, &sizelen, NULL, 0) < 0) { + VMBUS_LOG(ERR, + "could not read %s", sysctlVar); + goto error; + } + /* put the GPAD value in physical address */ + res->phys_addr = gpad; + } + } + return 0; +error: + return -1; +} + +int +vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, + struct mapped_vmbus_resource **uio_res) +{ + char devname[PATH_MAX]; /* contains the /dev/hv_uioX */ + + /* save fd if in primary process */ + snprintf(devname, sizeof(devname), "/dev/hv_uio%u", dev->uio_num); + dev->intr_handle.fd = open(devname, O_RDWR); + if (dev->intr_handle.fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + devname, strerror(errno)); + goto error; + } + dev->intr_handle.type = RTE_INTR_HANDLE_UIO_INTX; + + /* allocate the mapping details for secondary processes*/ + *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); + if (*uio_res == NULL) { + VMBUS_LOG(ERR, "cannot store uio mmap details"); + goto error; + } + + strlcpy((*uio_res)->path, devname, PATH_MAX); + rte_uuid_copy((*uio_res)->id, dev->device_id); + + if (sysctl_get_vmbus_device_info(dev) < 0) + goto error; + + return 0; +error: + vmbus_uio_free_resource(dev, *uio_res); + return -1; +} -- 2.30.2
Map the memory region created by hv_uio driver by passing their index number Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index 2059ab192b..41522ba2b5 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -5,6 +5,7 @@ #include <unistd.h> #include <fcntl.h> +#include <sys/mman.h> #include <sys/types.h> #include <sys/sysctl.h> #include <sys/ioctl.h> @@ -17,6 +18,7 @@ #include "private.h" const char *driver_name = "hv_uio"; +static void *vmbus_map_addr; /* Check map names with kernel names */ static const char *map_names[VMBUS_MAX_RESOURCE] = { @@ -140,3 +142,71 @@ vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, vmbus_uio_free_resource(dev, *uio_res); return -1; } + +static int +find_max_end_va(const struct rte_memseg_list *msl, void *arg) +{ + size_t sz = msl->memseg_arr.len * msl->page_sz; + void *end_va = RTE_PTR_ADD(msl->base_va, sz); + void **max_va = arg; + + if (*max_va < end_va) + *max_va = end_va; + return 0; +} + +/* + * TODO: this should be part of memseg api. + * code is duplicated from PCI. + */ +static void * +vmbus_find_max_end_va(void) +{ + void *va = NULL; + + rte_memseg_list_walk(find_max_end_va, &va); + return va; +} + +int +vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int idx, + struct mapped_vmbus_resource *uio_res, + int flags) +{ + size_t size = dev->resource[idx].len; + struct vmbus_map *maps = uio_res->maps; + void *mapaddr; + off_t offset; + int fd; + + /* devname for mmap */ + fd = open(uio_res->path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + uio_res->path, strerror(errno)); + return -1; + } + + /* try mapping somewhere close to the end of hugepages */ + if (vmbus_map_addr == NULL) + vmbus_map_addr = vmbus_find_max_end_va(); + + /* offset is special in uio it indicates which resource */ + offset = idx * rte_mem_page_size(); + + mapaddr = vmbus_map_resource(vmbus_map_addr, fd, offset, size, flags); + close(fd); + + if (mapaddr == MAP_FAILED) + return -1; + + dev->resource[idx].addr = mapaddr; + vmbus_map_addr = RTE_PTR_ADD(mapaddr, size); + + /* Record result of successful mapping for use by secondary */ + maps[idx].addr = mapaddr; + maps[idx].size = size; + + return 0; +} + -- 2.30.2
Event monitoring is not yet supported on FreeBSD, hence moving it to the OS specific files Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/freebsd/hn_os.c | 16 ++++++++++++++++ drivers/net/netvsc/hn_ethdev.c | 7 +++---- drivers/net/netvsc/hn_os.h | 6 ++++++ drivers/net/netvsc/linux/hn_os.c | 21 +++++++++++++++++++++ 4 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 drivers/net/netvsc/freebsd/hn_os.c create mode 100644 drivers/net/netvsc/hn_os.h create mode 100644 drivers/net/netvsc/linux/hn_os.c diff --git a/drivers/net/netvsc/freebsd/hn_os.c b/drivers/net/netvsc/freebsd/hn_os.c new file mode 100644 index 0000000000..4c6a79872d --- /dev/null +++ b/drivers/net/netvsc/freebsd/hn_os.c @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2021 Microsoft Corporation + */ + +#include <stdio.h> + +#include <rte_common.h> + +#include "hn_logs.h" +#include "hn_os.h" + +int eth_hn_os_dev_event(void) +{ + PMD_DRV_LOG(DEBUG, "rte_dev_event_monitor_start not supported on FreeBSD"); + return 0; +} diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c index 8a950403ac..8b1e07b775 100644 --- a/drivers/net/netvsc/hn_ethdev.c +++ b/drivers/net/netvsc/hn_ethdev.c @@ -39,6 +39,7 @@ #include "hn_rndis.h" #include "hn_nvs.h" #include "ndis.h" +#include "hn_os.h" #define HN_TX_OFFLOAD_CAPS (RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | \ RTE_ETH_TX_OFFLOAD_TCP_CKSUM | \ @@ -1240,11 +1241,9 @@ static int eth_hn_probe(struct rte_vmbus_driver *drv __rte_unused, PMD_INIT_FUNC_TRACE(); - ret = rte_dev_event_monitor_start(); - if (ret) { - PMD_DRV_LOG(ERR, "Failed to start device event monitoring"); + ret = eth_hn_os_dev_event(); + if (ret) return ret; - } eth_dev = eth_dev_vmbus_allocate(dev, sizeof(struct hn_data)); if (!eth_dev) diff --git a/drivers/net/netvsc/hn_os.h b/drivers/net/netvsc/hn_os.h new file mode 100644 index 0000000000..618c53cdcd --- /dev/null +++ b/drivers/net/netvsc/hn_os.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2009-2021 Microsoft Corp. + * All rights reserved. + */ + +int eth_hn_os_dev_event(void); diff --git a/drivers/net/netvsc/linux/hn_os.c b/drivers/net/netvsc/linux/hn_os.c new file mode 100644 index 0000000000..1ea12ce928 --- /dev/null +++ b/drivers/net/netvsc/linux/hn_os.c @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2021 Microsoft Corporation + */ + +#include <unistd.h> + +#include <rte_ethdev.h> + +#include "hn_logs.h" +#include "hn_os.h" + +int eth_hn_os_dev_event(void) +{ + int ret; + + ret = rte_dev_event_monitor_start(); + if (ret) + PMD_DRV_LOG(ERR, "Failed to start device event monitoring"); + + return ret; +} -- 2.30.2
Based on its channel id, mapping of primary channel or subchannel is determined. The primary channel memmory is mapped in the same way as done in Linux. For the subchannel an mmap request is directly made after determining the subchan memory offset Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 86 +++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index 41522ba2b5..b0c011dd9a 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -17,6 +17,12 @@ #include "private.h" +/* Macros to distinguish mmap request + * [7-0] - Device memory region + * [15-8]- Sub-channel id + */ +#define UH_SUBCHAN_MASK_SHIFT 8 + const char *driver_name = "hv_uio"; static void *vmbus_map_addr; @@ -210,3 +216,83 @@ vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int idx, return 0; } +static int vmbus_uio_map_primary(struct vmbus_channel *chan, + void **ring_buf, uint32_t *ring_size) +{ + struct mapped_vmbus_resource *uio_res; + + uio_res = vmbus_uio_find_resource(chan->device); + if (!uio_res) { + VMBUS_LOG(ERR, "can not find resources!"); + return -ENOMEM; + } + + if (uio_res->nb_maps < VMBUS_MAX_RESOURCE) { + VMBUS_LOG(ERR, "VMBUS: only %u resources found!", + uio_res->nb_maps); + return -EINVAL; + } + + *ring_size = uio_res->maps[HV_TXRX_RING_MAP].size / 2; + *ring_buf = uio_res->maps[HV_TXRX_RING_MAP].addr; + return 0; +} + +static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev, + struct vmbus_channel *chan, + void **ring_buf, uint32_t *ring_size) +{ + char ring_path[PATH_MAX]; + size_t size; + void *mapaddr; + off_t offset; + int fd; + + snprintf(ring_path, sizeof(ring_path), + "/dev/hv_uio%d", dev->uio_num); + + fd = open(ring_path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + ring_path, strerror(errno)); + return -errno; + } + + /* subchannel rings are of the same size as primary */ + size = dev->resource[HV_TXRX_RING_MAP].len; + offset = (chan->relid << UH_SUBCHAN_MASK_SHIFT) * PAGE_SIZE; + + mapaddr = vmbus_map_resource(vmbus_map_addr, fd, + offset, size, 0); + close(fd); + + if (mapaddr == MAP_FAILED) + return -EIO; + + *ring_size = size / 2; + *ring_buf = mapaddr; + + vmbus_map_addr = RTE_PTR_ADD(mapaddr, size); + return 0; +} + +int vmbus_uio_map_rings(struct vmbus_channel *chan) +{ + const struct rte_vmbus_device *dev = chan->device; + uint32_t ring_size; + void *ring_buf; + int ret; + + /* Primary channel */ + if (chan->subchannel_id == 0) + ret = vmbus_uio_map_primary(chan, &ring_buf, &ring_size); + else + ret = vmbus_uio_map_subchan(dev, chan, &ring_buf, &ring_size); + + if (ret) + return ret; + + vmbus_br_setup(&chan->txbr, ring_buf, ring_size); + vmbus_br_setup(&chan->rxbr, (char *)ring_buf + ring_size, ring_size); + return 0; +} -- 2.30.2
subchannels are always supported and so making vmbus_uio_subchannels_supported() always return true. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index b0c011dd9a..95b030c749 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -296,3 +296,11 @@ int vmbus_uio_map_rings(struct vmbus_channel *chan) vmbus_br_setup(&chan->rxbr, (char *)ring_buf + ring_size, ring_size); return 0; } + +bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan) +{ + RTE_SET_USED(dev); + RTE_SET_USED(chan); + return true; +} -- 2.30.2
In FreeBSD, unlike Linux there is no sub-channel open callback that could be called by HV_UIO driver, upon their grant by the hypervisor. Thus the PMD makes an IOCTL to the HV_UIO to open the sub-channels On Linux, the vmbus_uio_subchan_open() will always return success as the Linux HV_UIO opens them implicitly. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 30 +++++++++++++++++++++++++++ drivers/bus/vmbus/linux/vmbus_uio.c | 12 +++++++++++ drivers/bus/vmbus/private.h | 1 + drivers/bus/vmbus/rte_bus_vmbus.h | 10 +++++++++ drivers/bus/vmbus/version.map | 1 + drivers/bus/vmbus/vmbus_channel.c | 5 +++++ 6 files changed, 59 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index 95b030c749..affc0a3f4f 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -23,6 +23,9 @@ */ #define UH_SUBCHAN_MASK_SHIFT 8 +/* ioctl */ +#define HVIOOPENSUBCHAN _IOW('h', 14, uint32_t) + const char *driver_name = "hv_uio"; static void *vmbus_map_addr; @@ -304,3 +307,30 @@ bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, RTE_SET_USED(chan); return true; } + +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, uint32_t subchan) +{ + struct mapped_vmbus_resource *uio_res; + int fd, err = 0; + + uio_res = vmbus_uio_find_resource(dev); + if (!uio_res) { + VMBUS_LOG(ERR, "cannot find uio resource"); + return -EINVAL; + } + + fd = open(uio_res->path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + uio_res->path, strerror(errno)); + return -1; + } + + if (ioctl(fd, HVIOOPENSUBCHAN, &subchan)) { + VMBUS_LOG(ERR, "open subchan ioctl failed %s: %s", + uio_res->path, strerror(errno)); + err = -1; + } + close(fd); + return err; +} diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c b/drivers/bus/vmbus/linux/vmbus_uio.c index 5db70f8e0d..7241c052e7 100644 --- a/drivers/bus/vmbus/linux/vmbus_uio.c +++ b/drivers/bus/vmbus/linux/vmbus_uio.c @@ -471,3 +471,15 @@ int vmbus_uio_get_subchan(struct vmbus_channel *primary, closedir(chan_dir); return err; } + +/* + * This is a stub function and it should always succeed. + * The Linux UIO kernel driver opens the subchannels implicitly. + */ +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, + uint32_t subchan) +{ + RTE_SET_USED(dev); + RTE_SET_USED(subchan); + return 0; +} diff --git a/drivers/bus/vmbus/private.h b/drivers/bus/vmbus/private.h index 1bca147e12..ea0276a6c6 100644 --- a/drivers/bus/vmbus/private.h +++ b/drivers/bus/vmbus/private.h @@ -116,6 +116,7 @@ bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, int vmbus_uio_get_subchan(struct vmbus_channel *primary, struct vmbus_channel **subchan); int vmbus_uio_map_rings(struct vmbus_channel *chan); +int vmbus_uio_subchan_open(struct rte_vmbus_device *device, uint32_t subchan); void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen); diff --git a/drivers/bus/vmbus/rte_bus_vmbus.h b/drivers/bus/vmbus/rte_bus_vmbus.h index a24bad831d..c8cb4bc98c 100644 --- a/drivers/bus/vmbus/rte_bus_vmbus.h +++ b/drivers/bus/vmbus/rte_bus_vmbus.h @@ -404,6 +404,16 @@ void rte_vmbus_chan_dump(FILE *f, const struct vmbus_channel *chan); */ void rte_vmbus_unregister(struct rte_vmbus_driver *driver); +/** + * Perform IOCTL to VMBUS device + * + * @param device + * A pointer to a rte_vmbus_device structure + * @param subchan + * Count of subchannels to open + */ +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan); + /** Helper for VMBUS device registration from driver instance */ #define RTE_PMD_REGISTER_VMBUS(nm, vmbus_drv) \ RTE_INIT(vmbusinitfn_ ##nm) \ diff --git a/drivers/bus/vmbus/version.map b/drivers/bus/vmbus/version.map index 3cadec7fae..3509d4fc14 100644 --- a/drivers/bus/vmbus/version.map +++ b/drivers/bus/vmbus/version.map @@ -23,6 +23,7 @@ DPDK_22 { rte_vmbus_subchan_open; rte_vmbus_unmap_device; rte_vmbus_unregister; + rte_vmbus_ioctl; local: *; }; diff --git a/drivers/bus/vmbus/vmbus_channel.c b/drivers/bus/vmbus/vmbus_channel.c index 119b9b367e..9a8f6e3eef 100644 --- a/drivers/bus/vmbus/vmbus_channel.c +++ b/drivers/bus/vmbus/vmbus_channel.c @@ -365,6 +365,11 @@ int rte_vmbus_max_channels(const struct rte_vmbus_device *device) return 1; } +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan) +{ + return vmbus_uio_subchan_open(device, subchan); +} + /* Setup secondary channel */ int rte_vmbus_subchan_open(struct vmbus_channel *primary, struct vmbus_channel **new_chan) -- 2.30.2
make IOCTL call to open subchannels Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/hn_ethdev.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c index 8b1e07b775..104c7aebc5 100644 --- a/drivers/net/netvsc/hn_ethdev.c +++ b/drivers/net/netvsc/hn_ethdev.c @@ -516,6 +516,10 @@ static int hn_subchan_configure(struct hn_data *hv, if (err) return err; + err = rte_vmbus_ioctl(hv->vmbus, subchan); + if (err) + return err; + while (subchan > 0) { struct vmbus_channel *new_sc; uint16_t chn_index; -- 2.30.2
Using sysctl, all the subchannel information is fetched Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 85 +++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index affc0a3f4f..ca6f5fac59 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -308,6 +308,91 @@ bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, return true; } +static bool vmbus_isnew_subchannel(struct vmbus_channel *primary, + uint16_t id) +{ + const struct vmbus_channel *c; + + STAILQ_FOREACH(c, &primary->subchannel_list, next) { + if (c->relid == id) + return false; + } + return true; +} + +int vmbus_uio_get_subchan(struct vmbus_channel *primary, + struct vmbus_channel **subchan) +{ + const struct rte_vmbus_device *dev = primary->device; + char sysctlBuffer[PATH_MAX], sysctlVar[PATH_MAX]; + size_t len = PATH_MAX, sysctl_len; + /* nr_schan, relid, subid & monid datatype must match kernel's for sysctl */ + uint32_t relid, subid, nr_schan, i; + uint8_t monid; + int err; + + /* get no. of sub-channels opened by hv_uio + * dev.hv_uio.0.subchan_cnt + */ + snprintf(sysctlVar, len, "dev.%s.%d.subchan_cnt", driver_name, + dev->uio_num); + sysctl_len = sizeof(nr_schan); + if (sysctlbyname(sysctlVar, &nr_schan, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctlVar, + strerror(errno)); + return -1; + } + + /* dev.hv_uio.0.channel.14.sub */ + snprintf(sysctlBuffer, len, "dev.%s.%d.channel.%u.sub", driver_name, + dev->uio_num, primary->relid); + for (i = 1; i <= nr_schan; i++) { + /* get relid */ + snprintf(sysctlVar, len, "%s.%u.chanid", sysctlBuffer, i); + sysctl_len = sizeof(relid); + if (sysctlbyname(sysctlVar, &relid, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctlVar, + strerror(errno)); + goto error; + } + + if (!vmbus_isnew_subchannel(primary, (uint16_t)relid)) { + VMBUS_LOG(DEBUG, "skip already found channel: %u", + relid); + continue; + } + + /* get sub-channel id */ + snprintf(sysctlVar, len, "%s.%u.ch_subidx", sysctlBuffer, i); + sysctl_len = sizeof(subid); + if (sysctlbyname(sysctlVar, &subid, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctlVar, + strerror(errno)); + goto error; + } + + /* get monitor id */ + snprintf(sysctlVar, len, "%s.%u.monitor_id", sysctlBuffer, i); + sysctl_len = sizeof(monid); + if (sysctlbyname(sysctlVar, &monid, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctlVar, + strerror(errno)); + goto error; + } + + err = vmbus_chan_create(dev, (uint16_t)relid, (uint16_t)subid, + monid, subchan); + if (err) { + VMBUS_LOG(ERR, "subchannel setup failed"); + return err; + } + break; + } + return 0; +error: + return -1; +} + int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, uint32_t subchan) { struct mapped_vmbus_resource *uio_res; -- 2.30.2
Moved netvsc_hotplug_retry to respective OS dir as it contains OS dependent code. For Linux, it is copied as is and for FreeBSD it is not supported yet. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/freebsd/hn_os.c | 6 ++ drivers/net/netvsc/hn_ethdev.c | 84 ---------------------------- drivers/net/netvsc/hn_os.h | 2 + drivers/net/netvsc/linux/hn_os.c | 90 ++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 84 deletions(-) diff --git a/drivers/net/netvsc/freebsd/hn_os.c b/drivers/net/netvsc/freebsd/hn_os.c index 4c6a79872d..2ba4c32a76 100644 --- a/drivers/net/netvsc/freebsd/hn_os.c +++ b/drivers/net/netvsc/freebsd/hn_os.c @@ -14,3 +14,9 @@ int eth_hn_os_dev_event(void) PMD_DRV_LOG(DEBUG, "rte_dev_event_monitor_start not supported on FreeBSD"); return 0; } + +void netvsc_hotplug_retry(void *args) +{ + RTE_SET_USED(args); + return; +} diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c index 104c7aebc5..dd4b872fed 100644 --- a/drivers/net/netvsc/hn_ethdev.c +++ b/drivers/net/netvsc/hn_ethdev.c @@ -57,9 +57,6 @@ #define NETVSC_ARG_TXBREAK "tx_copybreak" #define NETVSC_ARG_RX_EXTMBUF_ENABLE "rx_extmbuf_enable" -/* The max number of retry when hot adding a VF device */ -#define NETVSC_MAX_HOTADD_RETRY 10 - struct hn_xstats_name_off { char name[RTE_ETH_XSTATS_NAME_SIZE]; unsigned int offset; @@ -556,87 +553,6 @@ static int hn_subchan_configure(struct hn_data *hv, return err; } -static void netvsc_hotplug_retry(void *args) -{ - int ret; - struct hn_data *hv = args; - struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id]; - struct rte_devargs *d = &hv->devargs; - char buf[256]; - - DIR *di; - struct dirent *dir; - struct ifreq req; - struct rte_ether_addr eth_addr; - int s; - - PMD_DRV_LOG(DEBUG, "%s: retry count %d", - __func__, hv->eal_hot_plug_retry); - - if (hv->eal_hot_plug_retry++ > NETVSC_MAX_HOTADD_RETRY) - return; - - snprintf(buf, sizeof(buf), "/sys/bus/pci/devices/%s/net", d->name); - di = opendir(buf); - if (!di) { - PMD_DRV_LOG(DEBUG, "%s: can't open directory %s, " - "retrying in 1 second", __func__, buf); - goto retry; - } - - while ((dir = readdir(di))) { - /* Skip . and .. directories */ - if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) - continue; - - /* trying to get mac address if this is a network device*/ - s = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); - if (s == -1) { - PMD_DRV_LOG(ERR, "Failed to create socket errno %d", - errno); - break; - } - strlcpy(req.ifr_name, dir->d_name, sizeof(req.ifr_name)); - ret = ioctl(s, SIOCGIFHWADDR, &req); - close(s); - if (ret == -1) { - PMD_DRV_LOG(ERR, - "Failed to send SIOCGIFHWADDR for device %s", - dir->d_name); - break; - } - if (req.ifr_hwaddr.sa_family != ARPHRD_ETHER) { - closedir(di); - return; - } - memcpy(eth_addr.addr_bytes, req.ifr_hwaddr.sa_data, - RTE_DIM(eth_addr.addr_bytes)); - - if (rte_is_same_ether_addr(ð_addr, dev->data->mac_addrs)) { - PMD_DRV_LOG(NOTICE, - "Found matching MAC address, adding device %s network name %s", - d->name, dir->d_name); - ret = rte_eal_hotplug_add(d->bus->name, d->name, - d->args); - if (ret) { - PMD_DRV_LOG(ERR, - "Failed to add PCI device %s", - d->name); - break; - } - } - /* When the code reaches here, we either have already added - * the device, or its MAC address did not match. - */ - closedir(di); - return; - } - closedir(di); -retry: - /* The device is still being initialized, retry after 1 second */ - rte_eal_alarm_set(1000000, netvsc_hotplug_retry, hv); -} - static void netvsc_hotadd_callback(const char *device_name, enum rte_dev_event_type type, void *arg) diff --git a/drivers/net/netvsc/hn_os.h b/drivers/net/netvsc/hn_os.h index 618c53cdcd..1fb7292b17 100644 --- a/drivers/net/netvsc/hn_os.h +++ b/drivers/net/netvsc/hn_os.h @@ -4,3 +4,5 @@ */ int eth_hn_os_dev_event(void); + +void netvsc_hotplug_retry(void *args); diff --git a/drivers/net/netvsc/linux/hn_os.c b/drivers/net/netvsc/linux/hn_os.c index 1ea12ce928..9c2f4cd7a8 100644 --- a/drivers/net/netvsc/linux/hn_os.c +++ b/drivers/net/netvsc/linux/hn_os.c @@ -3,12 +3,21 @@ */ #include <unistd.h> +#include <dirent.h> +#include <net/if.h> +#include <net/if_arp.h> +#include <sys/ioctl.h> #include <rte_ethdev.h> +#include <rte_alarm.h> #include "hn_logs.h" +#include "hn_var.h" #include "hn_os.h" +/* The max number of retry when hot adding a VF device */ +#define NETVSC_MAX_HOTADD_RETRY 10 + int eth_hn_os_dev_event(void) { int ret; @@ -19,3 +28,84 @@ int eth_hn_os_dev_event(void) return ret; } + +void netvsc_hotplug_retry(void *args) +{ + int ret; + struct hn_data *hv = args; + struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id]; + struct rte_devargs *d = &hv->devargs; + char buf[256]; + + DIR *di; + struct dirent *dir; + struct ifreq req; + struct rte_ether_addr eth_addr; + int s; + + PMD_DRV_LOG(DEBUG, "%s: retry count %d", + __func__, hv->eal_hot_plug_retry); + + if (hv->eal_hot_plug_retry++ > NETVSC_MAX_HOTADD_RETRY) + return; + + snprintf(buf, sizeof(buf), "/sys/bus/pci/devices/%s/net", d->name); + di = opendir(buf); + if (!di) { + PMD_DRV_LOG(DEBUG, "%s: can't open directory %s, " + "retrying in 1 second", __func__, buf); + goto retry; + } + + while ((dir = readdir(di))) { + /* Skip . and .. directories */ + if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) + continue; + + /* trying to get mac address if this is a network device*/ + s = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); + if (s == -1) { + PMD_DRV_LOG(ERR, "Failed to create socket errno %d", + errno); + break; + } + strlcpy(req.ifr_name, dir->d_name, sizeof(req.ifr_name)); + ret = ioctl(s, SIOCGIFHWADDR, &req); + close(s); + if (ret == -1) { + PMD_DRV_LOG(ERR, + "Failed to send SIOCGIFHWADDR for device %s", + dir->d_name); + break; + } + if (req.ifr_hwaddr.sa_family != ARPHRD_ETHER) { + closedir(di); + return; + } + memcpy(eth_addr.addr_bytes, req.ifr_hwaddr.sa_data, + RTE_DIM(eth_addr.addr_bytes)); + + if (rte_is_same_ether_addr(ð_addr, dev->data->mac_addrs)) { + PMD_DRV_LOG(NOTICE, + "Found matching MAC address, adding device %s network name %s", + d->name, dir->d_name); + ret = rte_eal_hotplug_add(d->bus->name, d->name, + d->args); + if (ret) { + PMD_DRV_LOG(ERR, + "Failed to add PCI device %s", + d->name); + break; + } + } + /* When the code reaches here, we either have already added + * the device, or its MAC address did not match. + */ + closedir(di); + return; + } + closedir(di); +retry: + /* The device is still being initialized, retry after 1 second */ + rte_eal_alarm_set(1000000, netvsc_hotplug_retry, hv); +} -- 2.30.2
add meson support for FreeBSD OS Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/meson.build | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build index 3892cbf67f..8fbe144fcd 100644 --- a/drivers/bus/vmbus/meson.build +++ b/drivers/bus/vmbus/meson.build @@ -20,7 +20,11 @@ if is_linux sources += files('linux/vmbus_bus.c', 'linux/vmbus_uio.c') includes += include_directories('linux') +elif is_freebsd + sources += files('freebsd/vmbus_bus.c', + 'freebsd/vmbus_uio.c') + includes += include_directories('freebsd') else build = false - reason = 'only supported on Linux' + reason = 'only supported on Linux & FreeBSD' endif -- 2.30.2
add meson support for FreeBSD Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/freebsd/meson.build | 6 ++++++ drivers/net/netvsc/linux/meson.build | 6 ++++++ drivers/net/netvsc/meson.build | 3 +++ 3 files changed, 15 insertions(+) create mode 100644 drivers/net/netvsc/freebsd/meson.build create mode 100644 drivers/net/netvsc/linux/meson.build diff --git a/drivers/net/netvsc/freebsd/meson.build b/drivers/net/netvsc/freebsd/meson.build new file mode 100644 index 0000000000..78f824f701 --- /dev/null +++ b/drivers/net/netvsc/freebsd/meson.build @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Microsoft Corporation + +sources += files( + 'hn_os.c', +) diff --git a/drivers/net/netvsc/linux/meson.build b/drivers/net/netvsc/linux/meson.build new file mode 100644 index 0000000000..78f824f701 --- /dev/null +++ b/drivers/net/netvsc/linux/meson.build @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Microsoft Corporation + +sources += files( + 'hn_os.c', +) diff --git a/drivers/net/netvsc/meson.build b/drivers/net/netvsc/meson.build index 399400dd01..1920413fb6 100644 --- a/drivers/net/netvsc/meson.build +++ b/drivers/net/netvsc/meson.build @@ -7,6 +7,8 @@ if is_windows subdir_done() endif +includes += include_directories(exec_env) + deps += 'bus_vmbus' sources = files( 'hn_ethdev.c', @@ -16,3 +18,4 @@ sources = files( 'hn_vf.c', ) +subdir(exec_env) -- 2.30.2
the IRQ masking/unmasking APIs use the HV_UIO driver's read and write CBs for their work. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_bus.c | 15 ++++++++++++++ drivers/bus/vmbus/freebsd/vmbus_uio.c | 28 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_bus.c b/drivers/bus/vmbus/freebsd/vmbus_bus.c index fa0f4883f7..b275723bd3 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_bus.c +++ b/drivers/bus/vmbus/freebsd/vmbus_bus.c @@ -265,3 +265,18 @@ rte_vmbus_scan(void) error: return -1; } + +void rte_vmbus_irq_mask(struct rte_vmbus_device *device) +{ + vmbus_uio_irq_control(device, 1); +} + +void rte_vmbus_irq_unmask(struct rte_vmbus_device *device) +{ + vmbus_uio_irq_control(device, 0); +} + +int rte_vmbus_irq_read(struct rte_vmbus_device *device) +{ + return vmbus_uio_irq_read(device); +} diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index ca6f5fac59..7eb8b73e29 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -38,6 +38,34 @@ static const char *map_names[VMBUS_MAX_RESOURCE] = { [HV_SEND_BUF_MAP] = "send_buf", }; +/* Control interrupts */ +void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff) +{ + if (write(dev->intr_handle.fd, &onoff, sizeof(onoff)) < 0) { + VMBUS_LOG(ERR, "cannot write to %d:%s", + dev->intr_handle.fd, strerror(errno)); + } +} + +int vmbus_uio_irq_read(struct rte_vmbus_device *dev) +{ + int32_t count; + int cc; + + cc = read(dev->intr_handle.fd, &count, sizeof(count)); + if (cc < (int)sizeof(count)) { + if (cc < 0) { + VMBUS_LOG(ERR, "IRQ read failed %s", + strerror(errno)); + return -errno; + } + VMBUS_LOG(ERR, "can't read IRQ count"); + return -EINVAL; + } + + return count; +} + void vmbus_uio_free_resource(struct rte_vmbus_device *dev, struct mapped_vmbus_resource *uio_res) -- 2.30.2
updated MAINTAINERS and doc files for FreeBSD support Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- MAINTAINERS | 2 ++ doc/guides/nics/netvsc.rst | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index d5cd0a6c2f..03bafd5b9b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -566,6 +566,7 @@ F: app/test/test_vdev.c VMBUS bus driver M: Stephen Hemminger <sthemmin@microsoft.com> M: Long Li <longli@microsoft.com> +M: Srikanth Kaka <srikanth.k@oneconvergence.com> F: drivers/bus/vmbus/ @@ -823,6 +824,7 @@ F: doc/guides/nics/vdev_netvsc.rst Microsoft Hyper-V netvsc M: Stephen Hemminger <sthemmin@microsoft.com> M: Long Li <longli@microsoft.com> +M: Srikanth Kaka <srikanth.k@oneconvergence.com> F: drivers/net/netvsc/ F: doc/guides/nics/netvsc.rst F: doc/guides/nics/features/netvsc.ini diff --git a/doc/guides/nics/netvsc.rst b/doc/guides/nics/netvsc.rst index 77efe1dc91..12d17024b7 100644 --- a/doc/guides/nics/netvsc.rst +++ b/doc/guides/nics/netvsc.rst @@ -91,6 +91,12 @@ operations: The dpdk-devbind.py script can not be used since it only handles PCI devices. +On FreeBSD, with hv_uio kernel driver loaded, do the following: + + .. code-block:: console + + devctl set driver -f hn1 hv_uio + Prerequisites ------------- @@ -101,6 +107,11 @@ The following prerequisites apply: Full support of multiple queues requires the 4.17 kernel. It is possible to use the netvsc PMD with 4.16 kernel but it is limited to a single queue. +* FreeBSD support for UIO on vmbus is done with hv_uio driver and it is still + in `review`_ + +.. _`review`: https://reviews.freebsd.org/D32184 + Netvsc PMD arguments -------------------- -- 2.30.2
On Thu, 17 Feb 2022 21:35:59 +0530
Srikanth Kaka <srikanth.k@oneconvergence.com> wrote:
> +/*
> + * GUID associated with network devices
> + * {f8615163-df3e-46c5-913f-f2d2f965ed0e}
> + */
> +static const rte_uuid_t vmbus_nic_uuid = {
> + 0xf8, 0x61, 0x51, 0x63,
> + 0xdf, 0x3e,
> + 0x46, 0xc5,
> + 0x91, 0x3f,
> + 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0xe
> +};
> +
> +extern struct rte_vmbus_bus rte_vmbus_bus;
> +
> +/* Parse UUID. Caller must pass NULL terminated string */
> +static int
> +parse_sysfs_uuid(const char *filename, rte_uuid_t uu)
> +{
> + char in[BUFSIZ];
> +
> + memcpy(in, filename, BUFSIZ);
> + if (rte_uuid_parse(in, uu) < 0) {
> + VMBUS_LOG(ERR, "%s not a valid UUID", in);
> + return -1;
> + }
> +
> + return 0;
> +}
> +
> +/* S
Since this code is copy/paste from the Linux version,
could you please put it in either common paraent directory
or in a new place unix/?
Two copies means bugs have to be fixed twice.
17/02/2022 17:05, Srikanth Kaka: > This patch series adds support to VMBUS & NetVSC PMDs run on FreeBSD In case you did not notice, this series is blocked by a CI compilation issue: http://mails.dpdk.org/archives/test-report/2022-February/261506.html
[-- Attachment #1: Type: text/plain, Size: 498 bytes --] Got it, thanks! The next patchset is in progress with the fix and it also removes common code between Linux and FreeBSD. It will be submitted soon. On Thu, 14 Apr 2022, 2:24 pm Thomas Monjalon, <thomas@monjalon.net> wrote: > 17/02/2022 17:05, Srikanth Kaka: > > This patch series adds support to VMBUS & NetVSC PMDs run on FreeBSD > > In case you did not notice, this series is blocked by a CI compilation > issue: > http://mails.dpdk.org/archives/test-report/2022-February/261506.html > > > > [-- Attachment #2: Type: text/html, Size: 1011 bytes --]
This patchset requires FreeBSD VMBus kernel changes and HV_UIO driver. Both are currently under review at https://reviews.freebsd.org/D32184 Changelog: v4: - moved OS independent code out of Linux v3: - split the patches into further logical parts - updated docs v2: - replaced strncpy with memcpy - replaced malloc.h with stdlib.h - added comment in linux/vmbus_uio.c v1: Intial release Srikanth Kaka (14): bus/vmbus: move independent code from Linux bus/vmbus: move independent bus functions bus/vmbus: move OS independent UIO functions bus/vmbus: scan and get the network device on FreeBSD bus/vmbus: handle mapping of device resources bus/vmbus: get device resource values using sysctl net/netvsc: make event monitor OS dependent bus/vmbus: add sub-channel mapping support bus/vmbus: open subchannels net/netvsc: make IOCTL call to open subchannels bus/vmbus: get subchannel info net/netvsc: moving hotplug retry to OS dir bus/vmbus: add meson support for FreeBSD bus/vmbus: update MAINTAINERS and docs MAINTAINERS | 2 + doc/guides/nics/netvsc.rst | 11 + drivers/bus/vmbus/freebsd/vmbus_bus.c | 286 ++++++++++++++++++++++ drivers/bus/vmbus/freebsd/vmbus_uio.c | 256 ++++++++++++++++++++ drivers/bus/vmbus/linux/vmbus_bus.c | 28 +-- drivers/bus/vmbus/linux/vmbus_uio.c | 320 +++---------------------- drivers/bus/vmbus/meson.build | 12 +- drivers/bus/vmbus/osi/vmbus_osi.h | 27 +++ drivers/bus/vmbus/osi/vmbus_osi_bus.c | 37 +++ drivers/bus/vmbus/osi/vmbus_osi_uio.c | 310 ++++++++++++++++++++++++ drivers/bus/vmbus/private.h | 1 + drivers/bus/vmbus/rte_bus_vmbus.h | 10 + drivers/bus/vmbus/version.map | 1 + drivers/bus/vmbus/vmbus_channel.c | 5 + drivers/net/netvsc/freebsd/hn_os.c | 21 ++ drivers/net/netvsc/freebsd/meson.build | 6 + drivers/net/netvsc/hn_ethdev.c | 95 +------- drivers/net/netvsc/hn_os.h | 8 + drivers/net/netvsc/linux/hn_os.c | 111 +++++++++ drivers/net/netvsc/linux/meson.build | 6 + drivers/net/netvsc/meson.build | 3 + 21 files changed, 1158 insertions(+), 398 deletions(-) create mode 100644 drivers/bus/vmbus/freebsd/vmbus_bus.c create mode 100644 drivers/bus/vmbus/freebsd/vmbus_uio.c create mode 100644 drivers/bus/vmbus/osi/vmbus_osi.h create mode 100644 drivers/bus/vmbus/osi/vmbus_osi_bus.c create mode 100644 drivers/bus/vmbus/osi/vmbus_osi_uio.c create mode 100644 drivers/net/netvsc/freebsd/hn_os.c create mode 100644 drivers/net/netvsc/freebsd/meson.build create mode 100644 drivers/net/netvsc/hn_os.h create mode 100644 drivers/net/netvsc/linux/hn_os.c create mode 100644 drivers/net/netvsc/linux/meson.build -- 2.30.2
Move the OS independent code from Linux dir in-order to be used by FreeBSD Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/linux/vmbus_bus.c | 13 +------------ drivers/bus/vmbus/meson.build | 5 +++++ drivers/bus/vmbus/osi/vmbus_osi.h | 11 +++++++++++ drivers/bus/vmbus/osi/vmbus_osi_bus.c | 20 ++++++++++++++++++++ 4 files changed, 37 insertions(+), 12 deletions(-) create mode 100644 drivers/bus/vmbus/osi/vmbus_osi.h create mode 100644 drivers/bus/vmbus/osi/vmbus_osi_bus.c diff --git a/drivers/bus/vmbus/linux/vmbus_bus.c b/drivers/bus/vmbus/linux/vmbus_bus.c index f502783f7a..c9a07041a7 100644 --- a/drivers/bus/vmbus/linux/vmbus_bus.c +++ b/drivers/bus/vmbus/linux/vmbus_bus.c @@ -21,22 +21,11 @@ #include "eal_filesystem.h" #include "private.h" +#include "vmbus_osi.h" /** Pathname of VMBUS devices directory. */ #define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" -/* - * GUID associated with network devices - * {f8615163-df3e-46c5-913f-f2d2f965ed0e} - */ -static const rte_uuid_t vmbus_nic_uuid = { - 0xf8, 0x61, 0x51, 0x63, - 0xdf, 0x3e, - 0x46, 0xc5, - 0x91, 0x3f, - 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0xe -}; - extern struct rte_vmbus_bus rte_vmbus_bus; /* Read sysfs file to get UUID */ diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build index 3892cbf67f..cbcba44e16 100644 --- a/drivers/bus/vmbus/meson.build +++ b/drivers/bus/vmbus/meson.build @@ -16,6 +16,11 @@ sources = files( 'vmbus_common_uio.c', ) +includes += include_directories('osi') +sources += files( + 'osi/vmbus_osi_bus.c' +) + if is_linux sources += files('linux/vmbus_bus.c', 'linux/vmbus_uio.c') diff --git a/drivers/bus/vmbus/osi/vmbus_osi.h b/drivers/bus/vmbus/osi/vmbus_osi.h new file mode 100644 index 0000000000..2db9399181 --- /dev/null +++ b/drivers/bus/vmbus/osi/vmbus_osi.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#ifndef _VMBUS_BUS_OSI_H_ +#define _VMBUS_BUS_OSI_H_ + +extern const rte_uuid_t vmbus_nic_uuid; + +#endif /* _VMBUS_BUS_OSI_H_ */ diff --git a/drivers/bus/vmbus/osi/vmbus_osi_bus.c b/drivers/bus/vmbus/osi/vmbus_osi_bus.c new file mode 100644 index 0000000000..8437109717 --- /dev/null +++ b/drivers/bus/vmbus/osi/vmbus_osi_bus.c @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <rte_uuid.h> + +#include "vmbus_osi.h" + +/* + * GUID associated with network devices + * {f8615163-df3e-46c5-913f-f2d2f965ed0e} + */ +const rte_uuid_t vmbus_nic_uuid = { + 0xf8, 0x61, 0x51, 0x63, + 0xdf, 0x3e, + 0x46, 0xc5, + 0x91, 0x3f, + 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0xe +}; -- 2.30.2
move independent Linux bus functions to OSI file Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/linux/vmbus_bus.c | 15 --------------- drivers/bus/vmbus/osi/vmbus_osi_bus.c | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/drivers/bus/vmbus/linux/vmbus_bus.c b/drivers/bus/vmbus/linux/vmbus_bus.c index c9a07041a7..030f18020e 100644 --- a/drivers/bus/vmbus/linux/vmbus_bus.c +++ b/drivers/bus/vmbus/linux/vmbus_bus.c @@ -358,18 +358,3 @@ rte_vmbus_scan(void) closedir(dir); return -1; } - -void rte_vmbus_irq_mask(struct rte_vmbus_device *device) -{ - vmbus_uio_irq_control(device, 1); -} - -void rte_vmbus_irq_unmask(struct rte_vmbus_device *device) -{ - vmbus_uio_irq_control(device, 0); -} - -int rte_vmbus_irq_read(struct rte_vmbus_device *device) -{ - return vmbus_uio_irq_read(device); -} diff --git a/drivers/bus/vmbus/osi/vmbus_osi_bus.c b/drivers/bus/vmbus/osi/vmbus_osi_bus.c index 8437109717..4aaa5459a8 100644 --- a/drivers/bus/vmbus/osi/vmbus_osi_bus.c +++ b/drivers/bus/vmbus/osi/vmbus_osi_bus.c @@ -3,8 +3,10 @@ * All Rights Reserved. */ +#include <rte_bus.h> #include <rte_uuid.h> +#include "private.h" #include "vmbus_osi.h" /* @@ -18,3 +20,18 @@ const rte_uuid_t vmbus_nic_uuid = { 0x91, 0x3f, 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0xe }; + +void rte_vmbus_irq_mask(struct rte_vmbus_device *device) +{ + vmbus_uio_irq_control(device, 1); +} + +void rte_vmbus_irq_unmask(struct rte_vmbus_device *device) +{ + vmbus_uio_irq_control(device, 0); +} + +int rte_vmbus_irq_read(struct rte_vmbus_device *device) +{ + return vmbus_uio_irq_read(device); +} -- 2.30.2
Moved all Linux independent UIO functions to OSI dir. Split the vmbus_uio_map_subchan() by keeping OS dependent code in vmbus_uio_map_subchan_os() function Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/linux/vmbus_uio.c | 292 +----------------------- drivers/bus/vmbus/meson.build | 3 +- drivers/bus/vmbus/osi/vmbus_osi.h | 12 + drivers/bus/vmbus/osi/vmbus_osi_uio.c | 306 ++++++++++++++++++++++++++ 4 files changed, 330 insertions(+), 283 deletions(-) create mode 100644 drivers/bus/vmbus/osi/vmbus_osi_uio.c diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c b/drivers/bus/vmbus/linux/vmbus_uio.c index 5db70f8e0d..ea6df21409 100644 --- a/drivers/bus/vmbus/linux/vmbus_uio.c +++ b/drivers/bus/vmbus/linux/vmbus_uio.c @@ -21,233 +21,18 @@ #include <rte_string_fns.h> #include "private.h" +#include "vmbus_osi.h" /** Pathname of VMBUS devices directory. */ #define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" -static void *vmbus_map_addr; - -/* Control interrupts */ -void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff) -{ - if ((rte_intr_fd_get(dev->intr_handle) < 0) || - write(rte_intr_fd_get(dev->intr_handle), &onoff, - sizeof(onoff)) < 0) { - VMBUS_LOG(ERR, "cannot write to %d:%s", - rte_intr_fd_get(dev->intr_handle), - strerror(errno)); - } -} - -int vmbus_uio_irq_read(struct rte_vmbus_device *dev) -{ - int32_t count; - int cc; - - if (rte_intr_fd_get(dev->intr_handle) < 0) - return -1; - - cc = read(rte_intr_fd_get(dev->intr_handle), &count, - sizeof(count)); - if (cc < (int)sizeof(count)) { - if (cc < 0) { - VMBUS_LOG(ERR, "IRQ read failed %s", - strerror(errno)); - return -errno; - } - VMBUS_LOG(ERR, "can't read IRQ count"); - return -EINVAL; - } - - return count; -} - -void -vmbus_uio_free_resource(struct rte_vmbus_device *dev, - struct mapped_vmbus_resource *uio_res) -{ - rte_free(uio_res); - - if (rte_intr_dev_fd_get(dev->intr_handle) >= 0) { - close(rte_intr_dev_fd_get(dev->intr_handle)); - rte_intr_dev_fd_set(dev->intr_handle, -1); - } - - if (rte_intr_fd_get(dev->intr_handle) >= 0) { - close(rte_intr_fd_get(dev->intr_handle)); - rte_intr_fd_set(dev->intr_handle, -1); - rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UNKNOWN); - } -} - -int -vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, - struct mapped_vmbus_resource **uio_res) -{ - char devname[PATH_MAX]; /* contains the /dev/uioX */ - int fd; - - /* save fd if in primary process */ - snprintf(devname, sizeof(devname), "/dev/uio%u", dev->uio_num); - fd = open(devname, O_RDWR); - if (fd < 0) { - VMBUS_LOG(ERR, "Cannot open %s: %s", - devname, strerror(errno)); - goto error; - } - - if (rte_intr_fd_set(dev->intr_handle, fd)) - goto error; - - if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UIO_INTX)) - goto error; - - /* allocate the mapping details for secondary processes*/ - *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); - if (*uio_res == NULL) { - VMBUS_LOG(ERR, "cannot store uio mmap details"); - goto error; - } - - strlcpy((*uio_res)->path, devname, PATH_MAX); - rte_uuid_copy((*uio_res)->id, dev->device_id); - - return 0; - -error: - vmbus_uio_free_resource(dev, *uio_res); - return -1; -} - -static int -find_max_end_va(const struct rte_memseg_list *msl, void *arg) -{ - size_t sz = msl->memseg_arr.len * msl->page_sz; - void *end_va = RTE_PTR_ADD(msl->base_va, sz); - void **max_va = arg; - - if (*max_va < end_va) - *max_va = end_va; - return 0; -} - -/* - * TODO: this should be part of memseg api. - * code is duplicated from PCI. - */ -static void * -vmbus_find_max_end_va(void) -{ - void *va = NULL; - - rte_memseg_list_walk(find_max_end_va, &va); - return va; -} - -int -vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int idx, - struct mapped_vmbus_resource *uio_res, - int flags) -{ - size_t size = dev->resource[idx].len; - struct vmbus_map *maps = uio_res->maps; - void *mapaddr; - off_t offset; - int fd; - - /* devname for mmap */ - fd = open(uio_res->path, O_RDWR); - if (fd < 0) { - VMBUS_LOG(ERR, "Cannot open %s: %s", - uio_res->path, strerror(errno)); - return -1; - } - - /* try mapping somewhere close to the end of hugepages */ - if (vmbus_map_addr == NULL) - vmbus_map_addr = vmbus_find_max_end_va(); - - /* offset is special in uio it indicates which resource */ - offset = idx * rte_mem_page_size(); - - mapaddr = vmbus_map_resource(vmbus_map_addr, fd, offset, size, flags); - close(fd); - - if (mapaddr == MAP_FAILED) - return -1; - - dev->resource[idx].addr = mapaddr; - vmbus_map_addr = RTE_PTR_ADD(mapaddr, size); - - /* Record result of successful mapping for use by secondary */ - maps[idx].addr = mapaddr; - maps[idx].size = size; - - return 0; -} - -static int vmbus_uio_map_primary(struct vmbus_channel *chan, - void **ring_buf, uint32_t *ring_size) -{ - struct mapped_vmbus_resource *uio_res; - - uio_res = vmbus_uio_find_resource(chan->device); - if (!uio_res) { - VMBUS_LOG(ERR, "can not find resources!"); - return -ENOMEM; - } - - if (uio_res->nb_maps < VMBUS_MAX_RESOURCE) { - VMBUS_LOG(ERR, "VMBUS: only %u resources found!", - uio_res->nb_maps); - return -EINVAL; - } - - *ring_size = uio_res->maps[HV_TXRX_RING_MAP].size / 2; - *ring_buf = uio_res->maps[HV_TXRX_RING_MAP].addr; - return 0; -} - -static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev, - const struct vmbus_channel *chan, - void **ring_buf, uint32_t *ring_size) +int vmbus_uio_map_subchan_os(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan, + void **mapaddr, size_t *file_size) { char ring_path[PATH_MAX]; - size_t file_size; struct stat sb; - void *mapaddr; int fd; - struct mapped_vmbus_resource *uio_res; - int channel_idx; - - uio_res = vmbus_uio_find_resource(dev); - if (!uio_res) { - VMBUS_LOG(ERR, "can not find resources for mapping subchan"); - return -ENOMEM; - } - - if (rte_eal_process_type() == RTE_PROC_PRIMARY) { - if (uio_res->nb_subchannels >= UIO_MAX_SUBCHANNEL) { - VMBUS_LOG(ERR, - "exceeding max subchannels UIO_MAX_SUBCHANNEL(%d)", - UIO_MAX_SUBCHANNEL); - VMBUS_LOG(ERR, "Change UIO_MAX_SUBCHANNEL and recompile"); - return -ENOMEM; - } - } else { - for (channel_idx = 0; channel_idx < uio_res->nb_subchannels; - channel_idx++) - if (uio_res->subchannel_maps[channel_idx].relid == - chan->relid) - break; - if (channel_idx == uio_res->nb_subchannels) { - VMBUS_LOG(ERR, - "couldn't find sub channel %d from shared mapping in primary", - chan->relid); - return -ENOMEM; - } - vmbus_map_addr = uio_res->subchannel_maps[channel_idx].addr; - } snprintf(ring_path, sizeof(ring_path), "%s/%s/channels/%u/ring", @@ -267,68 +52,23 @@ static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev, close(fd); return -errno; } - file_size = sb.st_size; + *file_size = sb.st_size; - if (file_size == 0 || (file_size & (rte_mem_page_size() - 1))) { + if (*file_size == 0 || (*file_size & (rte_mem_page_size() - 1))) { VMBUS_LOG(ERR, "incorrect size %s: %zu", - ring_path, file_size); + ring_path, *file_size); close(fd); return -EINVAL; } - mapaddr = vmbus_map_resource(vmbus_map_addr, fd, - 0, file_size, 0); + *mapaddr = vmbus_map_resource(vmbus_map_addr, fd, + 0, *file_size, 0); close(fd); - if (mapaddr == MAP_FAILED) + if (*mapaddr == MAP_FAILED) return -EIO; - if (rte_eal_process_type() == RTE_PROC_PRIMARY) { - - /* Add this mapping to uio_res for use by secondary */ - uio_res->subchannel_maps[uio_res->nb_subchannels].relid = - chan->relid; - uio_res->subchannel_maps[uio_res->nb_subchannels].addr = - mapaddr; - uio_res->subchannel_maps[uio_res->nb_subchannels].size = - file_size; - uio_res->nb_subchannels++; - - vmbus_map_addr = RTE_PTR_ADD(mapaddr, file_size); - } else { - if (mapaddr != vmbus_map_addr) { - VMBUS_LOG(ERR, "failed to map channel %d to addr %p", - chan->relid, mapaddr); - vmbus_unmap_resource(mapaddr, file_size); - return -EIO; - } - } - - *ring_size = file_size / 2; - *ring_buf = mapaddr; - - return 0; -} - -int vmbus_uio_map_rings(struct vmbus_channel *chan) -{ - const struct rte_vmbus_device *dev = chan->device; - uint32_t ring_size; - void *ring_buf; - int ret; - - /* Primary channel */ - if (chan->subchannel_id == 0) - ret = vmbus_uio_map_primary(chan, &ring_buf, &ring_size); - else - ret = vmbus_uio_map_subchan(dev, chan, &ring_buf, &ring_size); - - if (ret) - return ret; - - vmbus_br_setup(&chan->txbr, ring_buf, ring_size); - vmbus_br_setup(&chan->rxbr, (char *)ring_buf + ring_size, ring_size); return 0; } @@ -377,18 +117,6 @@ bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, return vmbus_uio_ring_present(dev, chan->relid); } -static bool vmbus_isnew_subchannel(struct vmbus_channel *primary, - unsigned long id) -{ - const struct vmbus_channel *c; - - STAILQ_FOREACH(c, &primary->subchannel_list, next) { - if (c->relid == id) - return false; - } - return true; -} - int vmbus_uio_get_subchan(struct vmbus_channel *primary, struct vmbus_channel **subchan) { diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build index cbcba44e16..fe9c72bce1 100644 --- a/drivers/bus/vmbus/meson.build +++ b/drivers/bus/vmbus/meson.build @@ -18,7 +18,8 @@ sources = files( includes += include_directories('osi') sources += files( - 'osi/vmbus_osi_bus.c' + 'osi/vmbus_osi_bus.c', + 'osi/vmbus_osi_uio.c' ) if is_linux diff --git a/drivers/bus/vmbus/osi/vmbus_osi.h b/drivers/bus/vmbus/osi/vmbus_osi.h index 2db9399181..579c4bb99c 100644 --- a/drivers/bus/vmbus/osi/vmbus_osi.h +++ b/drivers/bus/vmbus/osi/vmbus_osi.h @@ -6,6 +6,18 @@ #ifndef _VMBUS_BUS_OSI_H_ #define _VMBUS_BUS_OSI_H_ +#include <rte_bus_vmbus.h> + +#include "private.h" + extern const rte_uuid_t vmbus_nic_uuid; +extern void *vmbus_map_addr; + +int vmbus_uio_map_subchan_os(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan, + void **mapaddr, size_t *file_size); + +bool vmbus_isnew_subchannel(struct vmbus_channel *primary, + unsigned long id); #endif /* _VMBUS_BUS_OSI_H_ */ diff --git a/drivers/bus/vmbus/osi/vmbus_osi_uio.c b/drivers/bus/vmbus/osi/vmbus_osi_uio.c new file mode 100644 index 0000000000..35106e247e --- /dev/null +++ b/drivers/bus/vmbus/osi/vmbus_osi_uio.c @@ -0,0 +1,306 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <unistd.h> +#include <fcntl.h> +#include <sys/mman.h> + +#include <rte_eal.h> +#include <rte_log.h> +#include <rte_bus.h> +#include <rte_memory.h> +#include <rte_common.h> +#include <rte_malloc.h> +#include <rte_bus_vmbus.h> +#include <rte_string_fns.h> + +#include "private.h" +#include "vmbus_osi.h" + +void *vmbus_map_addr; + +/* Control interrupts */ +void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff) +{ + if ((rte_intr_fd_get(dev->intr_handle) < 0) || + write(rte_intr_fd_get(dev->intr_handle), &onoff, + sizeof(onoff)) < 0) { + VMBUS_LOG(ERR, "cannot write to %d:%s", + rte_intr_fd_get(dev->intr_handle), + strerror(errno)); + } +} + +int vmbus_uio_irq_read(struct rte_vmbus_device *dev) +{ + int32_t count; + int cc; + + if (rte_intr_fd_get(dev->intr_handle) < 0) + return -1; + + cc = read(rte_intr_fd_get(dev->intr_handle), &count, + sizeof(count)); + if (cc < (int)sizeof(count)) { + if (cc < 0) { + VMBUS_LOG(ERR, "IRQ read failed %s", + strerror(errno)); + return -errno; + } + VMBUS_LOG(ERR, "can't read IRQ count"); + return -EINVAL; + } + + return count; +} + +void +vmbus_uio_free_resource(struct rte_vmbus_device *dev, + struct mapped_vmbus_resource *uio_res) +{ + rte_free(uio_res); + + if (rte_intr_dev_fd_get(dev->intr_handle) >= 0) { + close(rte_intr_dev_fd_get(dev->intr_handle)); + rte_intr_dev_fd_set(dev->intr_handle, -1); + } + + if (rte_intr_fd_get(dev->intr_handle) >= 0) { + close(rte_intr_fd_get(dev->intr_handle)); + rte_intr_fd_set(dev->intr_handle, -1); + rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UNKNOWN); + } +} + +int +vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, + struct mapped_vmbus_resource **uio_res) +{ + char devname[PATH_MAX]; /* contains the /dev/uioX */ + int fd; + + /* save fd if in primary process */ + snprintf(devname, sizeof(devname), "/dev/uio%u", dev->uio_num); + fd = open(devname, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + devname, strerror(errno)); + goto error; + } + + if (rte_intr_fd_set(dev->intr_handle, fd)) + goto error; + + if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UIO_INTX)) + goto error; + + /* allocate the mapping details for secondary processes*/ + *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); + if (*uio_res == NULL) { + VMBUS_LOG(ERR, "cannot store uio mmap details"); + goto error; + } + + strlcpy((*uio_res)->path, devname, PATH_MAX); + rte_uuid_copy((*uio_res)->id, dev->device_id); + + return 0; + +error: + vmbus_uio_free_resource(dev, *uio_res); + return -1; +} + +static int +find_max_end_va(const struct rte_memseg_list *msl, void *arg) +{ + size_t sz = msl->memseg_arr.len * msl->page_sz; + void *end_va = RTE_PTR_ADD(msl->base_va, sz); + void **max_va = arg; + + if (*max_va < end_va) + *max_va = end_va; + return 0; +} + +/* + * TODO: this should be part of memseg api. + * code is duplicated from PCI. + */ +static void * +vmbus_find_max_end_va(void) +{ + void *va = NULL; + + rte_memseg_list_walk(find_max_end_va, &va); + return va; +} + +int +vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int idx, + struct mapped_vmbus_resource *uio_res, + int flags) +{ + size_t size = dev->resource[idx].len; + struct vmbus_map *maps = uio_res->maps; + void *mapaddr; + off_t offset; + int fd; + + /* devname for mmap */ + fd = open(uio_res->path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + uio_res->path, strerror(errno)); + return -1; + } + + /* try mapping somewhere close to the end of hugepages */ + if (vmbus_map_addr == NULL) + vmbus_map_addr = vmbus_find_max_end_va(); + + /* offset is special in uio it indicates which resource */ + offset = idx * rte_mem_page_size(); + + mapaddr = vmbus_map_resource(vmbus_map_addr, fd, offset, size, flags); + close(fd); + + if (mapaddr == MAP_FAILED) + return -1; + + dev->resource[idx].addr = mapaddr; + vmbus_map_addr = RTE_PTR_ADD(mapaddr, size); + + /* Record result of successful mapping for use by secondary */ + maps[idx].addr = mapaddr; + maps[idx].size = size; + + return 0; +} + +static int vmbus_uio_map_primary(struct vmbus_channel *chan, + void **ring_buf, uint32_t *ring_size) +{ + struct mapped_vmbus_resource *uio_res; + + uio_res = vmbus_uio_find_resource(chan->device); + if (!uio_res) { + VMBUS_LOG(ERR, "can not find resources!"); + return -ENOMEM; + } + + if (uio_res->nb_maps < VMBUS_MAX_RESOURCE) { + VMBUS_LOG(ERR, "VMBUS: only %u resources found!", + uio_res->nb_maps); + return -EINVAL; + } + + *ring_size = uio_res->maps[HV_TXRX_RING_MAP].size / 2; + *ring_buf = uio_res->maps[HV_TXRX_RING_MAP].addr; + return 0; +} + +static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan, + void **ring_buf, uint32_t *ring_size) +{ + size_t file_size = 0; + void *mapaddr = NULL; + struct mapped_vmbus_resource *uio_res; + int channel_idx; + int ret; + + uio_res = vmbus_uio_find_resource(dev); + if (!uio_res) { + VMBUS_LOG(ERR, "can not find resources for mapping subchan"); + return -ENOMEM; + } + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + if (uio_res->nb_subchannels >= UIO_MAX_SUBCHANNEL) { + VMBUS_LOG(ERR, + "exceeding max subchannels UIO_MAX_SUBCHANNEL(%d)", + UIO_MAX_SUBCHANNEL); + VMBUS_LOG(ERR, "Change UIO_MAX_SUBCHANNEL and recompile"); + return -ENOMEM; + } + } else { + for (channel_idx = 0; channel_idx < uio_res->nb_subchannels; + channel_idx++) + if (uio_res->subchannel_maps[channel_idx].relid == + chan->relid) + break; + if (channel_idx == uio_res->nb_subchannels) { + VMBUS_LOG(ERR, + "couldn't find sub channel %d from shared mapping in primary", + chan->relid); + return -ENOMEM; + } + vmbus_map_addr = uio_res->subchannel_maps[channel_idx].addr; + } + + ret = vmbus_uio_map_subchan_os(dev, chan, &mapaddr, &file_size); + if (ret) + return ret; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + + /* Add this mapping to uio_res for use by secondary */ + uio_res->subchannel_maps[uio_res->nb_subchannels].relid = + chan->relid; + uio_res->subchannel_maps[uio_res->nb_subchannels].addr = + mapaddr; + uio_res->subchannel_maps[uio_res->nb_subchannels].size = + file_size; + uio_res->nb_subchannels++; + + vmbus_map_addr = RTE_PTR_ADD(mapaddr, file_size); + } else { + if (mapaddr != vmbus_map_addr) { + VMBUS_LOG(ERR, "failed to map channel %d to addr %p", + chan->relid, mapaddr); + vmbus_unmap_resource(mapaddr, file_size); + return -EIO; + } + } + + *ring_size = file_size / 2; + *ring_buf = mapaddr; + + return 0; +} + +int vmbus_uio_map_rings(struct vmbus_channel *chan) +{ + const struct rte_vmbus_device *dev = chan->device; + uint32_t ring_size = 0; + void *ring_buf = NULL; + int ret; + + /* Primary channel */ + if (chan->subchannel_id == 0) + ret = vmbus_uio_map_primary(chan, &ring_buf, &ring_size); + else + ret = vmbus_uio_map_subchan(dev, chan, &ring_buf, &ring_size); + + if (ret) + return ret; + + vmbus_br_setup(&chan->txbr, ring_buf, ring_size); + vmbus_br_setup(&chan->rxbr, (char *)ring_buf + ring_size, ring_size); + return 0; +} + +bool vmbus_isnew_subchannel(struct vmbus_channel *primary, + unsigned long id) +{ + const struct vmbus_channel *c; + + STAILQ_FOREACH(c, &primary->subchannel_list, next) { + if (c->relid == id) + return false; + } + return true; +} -- 2.30.2
Using sysctl, all the devices on the VMBUS are identified by the PMD. On finding the Network device's device id, it is added to VMBUS dev list. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_bus.c | 268 ++++++++++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 drivers/bus/vmbus/freebsd/vmbus_bus.c diff --git a/drivers/bus/vmbus/freebsd/vmbus_bus.c b/drivers/bus/vmbus/freebsd/vmbus_bus.c new file mode 100644 index 0000000000..8cf3bc5a6a --- /dev/null +++ b/drivers/bus/vmbus/freebsd/vmbus_bus.c @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <rte_eal.h> +#include <rte_bus_vmbus.h> + +#include "private.h" +#include "vmbus_osi.h" + +#include <sys/bus.h> +#include <sys/types.h> +#include <sys/sysctl.h> + +/* Parse UUID. Caller must pass NULL terminated string */ +static int +parse_sysfs_uuid(const char *filename, rte_uuid_t uu) +{ + char in[BUFSIZ]; + + memcpy(in, filename, BUFSIZ); + if (rte_uuid_parse(in, uu) < 0) { + VMBUS_LOG(ERR, "%s not a valid UUID", in); + return -1; + } + + return 0; +} + +/* Scan one vmbus entry, and fill the devices list from it. */ +static int +vmbus_scan_one(const char *name, unsigned int unit_num) +{ + struct rte_vmbus_device *dev, *dev2; + char sysctlBuffer[PATH_MAX], sysctlVar[PATH_MAX]; + size_t guid_len = 36, len = PATH_MAX; + char classid[guid_len + 1], deviceid[guid_len + 1]; + + dev = calloc(1, sizeof(*dev)); + if (dev == NULL) + return -1; + + /* get class id and device id */ + snprintf(sysctlVar, len, "dev.%s.%u.%%pnpinfo", name, unit_num); + if (sysctlbyname(sysctlVar, &sysctlBuffer, &len, NULL, 0) < 0) + goto error; + + /* pnpinfo: classid=f912ad6d-2b17-48ea-bd65-f927a61c7684 + * deviceid=d34b2567-b9b6-42b9-8778-0a4ec0b955bf + */ + if (sysctlBuffer[0] == 'c' && sysctlBuffer[1] == 'l' && + sysctlBuffer[7] == '=') { + memcpy(classid, &sysctlBuffer[8], guid_len); + classid[guid_len] = '\0'; + } + if (parse_sysfs_uuid(classid, dev->class_id) < 0) + goto error; + + /* skip non-network devices */ + if (rte_uuid_compare(dev->class_id, vmbus_nic_uuid) != 0) { + free(dev); + return 0; + } + + if (sysctlBuffer[45] == 'd' && sysctlBuffer[46] == 'e' && + sysctlBuffer[47] == 'v' && sysctlBuffer[53] == '=') { + memcpy(deviceid, &sysctlBuffer[54], guid_len); + deviceid[guid_len] = '\0'; + } + if (parse_sysfs_uuid(deviceid, dev->device_id) < 0) + goto error; + + if (!strcmp(name, "hv_uio")) + dev->uio_num = unit_num; + else + dev->uio_num = -1; + dev->device.bus = &rte_vmbus_bus.bus; + dev->device.numa_node = 0; + dev->device.name = strdup(deviceid); + if (!dev->device.name) + goto error; + + dev->device.devargs = vmbus_devargs_lookup(dev); + + dev->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE); + if (dev->intr_handle == NULL) + goto error; + + /* device is valid, add in list (sorted) */ + VMBUS_LOG(DEBUG, "Adding vmbus device %s", name); + + TAILQ_FOREACH(dev2, &rte_vmbus_bus.device_list, next) { + int ret; + + ret = rte_uuid_compare(dev->device_id, dev2->device_id); + if (ret > 0) + continue; + + if (ret < 0) { + vmbus_insert_device(dev2, dev); + } else { /* already registered */ + VMBUS_LOG(NOTICE, + "%s already registered", name); + free(dev); + } + return 0; + } + + vmbus_add_device(dev); + return 0; +error: + VMBUS_LOG(DEBUG, "failed"); + + free(dev); + return -1; +} + +static int +vmbus_unpack(char *walker, char *ep, char **str) +{ + int ret = 0; + + *str = strdup(walker); + if (*str == NULL) { + ret = -ENOMEM; + goto exit; + } + + if (walker + strnlen(walker, ep - walker) >= ep) { + ret = -EINVAL; + goto exit; + } +exit: + return ret; +} + +/* + * Scan the content of the vmbus, and the devices in the devices list + */ +int +rte_vmbus_scan(void) +{ + struct u_device udev; + struct u_businfo ubus; + int dev_idx, dev_ptr, name2oid[2], oid[CTL_MAXNAME + 12], error; + size_t oidlen, rlen, ub_size; + uintptr_t vmbus_handle = 0; + char *walker, *ep; + char name[16] = "hw.bus.devices"; + char *dd_name, *dd_desc, *dd_drivername, *dd_pnpinfo, *dd_location; + + /* + * devinfo FreeBSD APP logic to fetch all the VMBus devices + * using SYSCTLs + */ + name2oid[0] = 0; + name2oid[1] = 3; + oidlen = sizeof(oid); + error = sysctl(name2oid, 2, oid, &oidlen, name, strlen(name)); + if (error < 0) { + VMBUS_LOG(DEBUG, "can't find hw.bus.devices sysctl node"); + return -ENOENT; + } + oidlen /= sizeof(int); + if (oidlen > CTL_MAXNAME) { + VMBUS_LOG(DEBUG, "hw.bus.devices oid is too large"); + return -EINVAL; + } + + ub_size = sizeof(ubus); + if (sysctlbyname("hw.bus.info", &ubus, &ub_size, NULL, 0) != 0) { + VMBUS_LOG(DEBUG, "sysctlbyname(\"hw.bus.info\", ...) failed"); + return -EINVAL; + } + if ((ub_size != sizeof(ubus)) || + (ubus.ub_version != BUS_USER_VERSION)) { + VMBUS_LOG(DEBUG, + "kernel bus interface version mismatch: kernel %d expected %d", + ubus.ub_version, BUS_USER_VERSION); + return -EINVAL; + } + + oid[oidlen++] = ubus.ub_generation; + dev_ptr = oidlen++; + + /* + * Scan devices. + * + * Stop after a fairly insane number to avoid death in the case + * of kernel corruption. + */ + + for (dev_idx = 0; dev_idx < 10000; dev_idx++) { + /* + * Get the device information. + */ + oid[dev_ptr] = dev_idx; + rlen = sizeof(udev); + error = sysctl(oid, oidlen, &udev, &rlen, NULL, 0); + if (error < 0) { + if (errno == ENOENT) /* end of list */ + break; + if (errno != EINVAL) /* gen count skip, restart */ + VMBUS_LOG(DEBUG, "sysctl hw.bus.devices.%d", + dev_idx); + return errno; + } + if (rlen != sizeof(udev)) { + VMBUS_LOG(DEBUG, + "sysctl returned wrong data %zd bytes instead of %zd", + rlen, sizeof(udev)); + return -EINVAL; + } + + walker = udev.dv_fields; + ep = walker + sizeof(udev.dv_fields); + dd_name = NULL; + dd_desc = NULL; + dd_drivername = NULL; + dd_pnpinfo = NULL; + dd_location = NULL; + + error = vmbus_unpack(walker, ep, &dd_name); + if (error < 0) + return error; + walker += strlen(walker) + 1; + + error = vmbus_unpack(walker, ep, &dd_desc); + if (error < 0) + return error; + walker += strlen(walker) + 1; + + error = vmbus_unpack(walker, ep, &dd_drivername); + if (error < 0) + return error; + walker += strlen(walker) + 1; + + error = vmbus_unpack(walker, ep, &dd_pnpinfo); + if (error < 0) + return error; + walker += strlen(walker) + 1; + + error = vmbus_unpack(walker, ep, &dd_location); + if (error < 0) + return error; + + if (*dd_drivername && !(strcmp(dd_drivername, "vmbus"))) + vmbus_handle = udev.dv_handle; + + if (vmbus_handle && (vmbus_handle == udev.dv_parent) + && *dd_pnpinfo && *dd_name) { + unsigned int driver_len = 0, unit_num = 0; + char *endptr; + + driver_len = strlen(dd_drivername); + unit_num = strtoull(&dd_name[driver_len], &endptr, 10); + VMBUS_LOG(DEBUG, "Device name:%s, pnpinfo:%s", + dd_name, dd_pnpinfo); + + if (vmbus_scan_one(dd_drivername, unit_num) < 0) + goto error; + } + } + return 0; +error: + return -1; +} -- 2.30.2
All resource values are published by HV_UIO driver as sysctl key value pairs and they are read at a later point of the code flow Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_bus.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_bus.c b/drivers/bus/vmbus/freebsd/vmbus_bus.c index 8cf3bc5a6a..afa1e74976 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_bus.c +++ b/drivers/bus/vmbus/freebsd/vmbus_bus.c @@ -28,6 +28,24 @@ parse_sysfs_uuid(const char *filename, rte_uuid_t uu) return 0; } +/* map the resources of a vmbus device in virtual memory */ +int +rte_vmbus_map_device(struct rte_vmbus_device *dev) +{ + if (dev->uio_num < 0) { + VMBUS_LOG(DEBUG, "Not managed by UIO driver, skipped"); + return 1; + } + + return vmbus_uio_map_resource(dev); +} + +void +rte_vmbus_unmap_device(struct rte_vmbus_device *dev) +{ + vmbus_uio_unmap_resource(dev); +} + /* Scan one vmbus entry, and fill the devices list from it. */ static int vmbus_scan_one(const char *name, unsigned int unit_num) -- 2.30.2
The UIO device's attribute (relid, monitor id, etc) values are retrieved using following sysctl variables: $ sysctl dev.hv_uio.0 dev.hv_uio.0.send_buf.gpadl: 925241 dev.hv_uio.0.send_buf.size: 16777216 dev.hv_uio.0.recv_buf.gpadl: 925240 dev.hv_uio.0.recv_buf.size: 32505856 dev.hv_uio.0.monitor_page.size: 4096 dev.hv_uio.0.int_page.size: 4096 Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 105 ++++++++++++++++++++++++++ drivers/bus/vmbus/linux/vmbus_uio.c | 16 ++++ drivers/bus/vmbus/osi/vmbus_osi.h | 4 + drivers/bus/vmbus/osi/vmbus_osi_uio.c | 6 +- 4 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 drivers/bus/vmbus/freebsd/vmbus_uio.c diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c new file mode 100644 index 0000000000..b622388ce9 --- /dev/null +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <unistd.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/types.h> +#include <sys/sysctl.h> +#include <sys/ioctl.h> + +#include <rte_log.h> +#include <rte_bus.h> +#include <rte_malloc.h> +#include <rte_bus_vmbus.h> + +#include "private.h" +#include "vmbus_osi.h" + +const char *driver_name = "hv_uio"; + +/* Check map names with kernel names */ +static const char *map_names[VMBUS_MAX_RESOURCE] = { + [HV_TXRX_RING_MAP] = "txrx_rings", + [HV_INT_PAGE_MAP] = "int_page", + [HV_MON_PAGE_MAP] = "monitor_page", + [HV_RECV_BUF_MAP] = "recv_buf", + [HV_SEND_BUF_MAP] = "send_buf", +}; + +static int +sysctl_get_vmbus_device_info(struct rte_vmbus_device *dev) +{ + char sysctlBuffer[PATH_MAX]; + char sysctlVar[PATH_MAX]; + size_t len = PATH_MAX, sysctl_len; + unsigned long tmp; + int i; + + snprintf(sysctlBuffer, len, "dev.%s.%d", driver_name, dev->uio_num); + + sysctl_len = sizeof(unsigned long); + /* get relid */ + snprintf(sysctlVar, len, "%s.channel.ch_id", sysctlBuffer); + if (sysctlbyname(sysctlVar, &tmp, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s", sysctlVar); + goto error; + } + dev->relid = tmp; + + /* get monitor id */ + snprintf(sysctlVar, len, "%s.channel.%u.monitor_id", sysctlBuffer, + dev->relid); + if (sysctlbyname(sysctlVar, &tmp, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s", sysctlVar); + goto error; + } + dev->monitor_id = tmp; + + /* Extract resource value */ + for (i = 0; i < VMBUS_MAX_RESOURCE; i++) { + struct rte_mem_resource *res = &dev->resource[i]; + unsigned long size, gpad = 0; + size_t sizelen = sizeof(len); + + snprintf(sysctlVar, sizeof(sysctlVar), "%s.%s.size", + sysctlBuffer, map_names[i]); + if (sysctlbyname(sysctlVar, &size, &sizelen, NULL, 0) < 0) { + VMBUS_LOG(ERR, + "could not read %s", sysctlVar); + goto error; + } + res->len = size; + + if (i == HV_RECV_BUF_MAP || i == HV_SEND_BUF_MAP) { + snprintf(sysctlVar, sizeof(sysctlVar), "%s.%s.gpadl", + sysctlBuffer, map_names[i]); + if (sysctlbyname(sysctlVar, &gpad, &sizelen, NULL, 0) < 0) { + VMBUS_LOG(ERR, + "could not read %s", sysctlVar); + goto error; + } + /* put the GPAD value in physical address */ + res->phys_addr = gpad; + } + } + return 0; +error: + return -1; +} + +/* + * On FreeBSD, the device is opened first to ensure kernel UIO driver + * is properly initialized before reading device attributes + */ +int vmbus_get_device_info_os(struct rte_vmbus_device *dev) +{ + return sysctl_get_vmbus_device_info(dev); +} + +const char *get_devname_os(void) +{ + return "/dev/hv_uio"; +} diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c b/drivers/bus/vmbus/linux/vmbus_uio.c index ea6df21409..669551a4d4 100644 --- a/drivers/bus/vmbus/linux/vmbus_uio.c +++ b/drivers/bus/vmbus/linux/vmbus_uio.c @@ -199,3 +199,19 @@ int vmbus_uio_get_subchan(struct vmbus_channel *primary, closedir(chan_dir); return err; } + +/* + * In Linux the device info is fetched from SYSFS and doesn't need + * opening of the device before reading its attributes + * This is a stub function and it should always succeed. + */ +int vmbus_get_device_info_os(struct rte_vmbus_device *dev) +{ + RTE_SET_USED(dev); + return 0; +} + +const char *get_devname_os(void) +{ + return "/dev/uio"; +} diff --git a/drivers/bus/vmbus/osi/vmbus_osi.h b/drivers/bus/vmbus/osi/vmbus_osi.h index 579c4bb99c..59afc102f1 100644 --- a/drivers/bus/vmbus/osi/vmbus_osi.h +++ b/drivers/bus/vmbus/osi/vmbus_osi.h @@ -20,4 +20,8 @@ int vmbus_uio_map_subchan_os(const struct rte_vmbus_device *dev, bool vmbus_isnew_subchannel(struct vmbus_channel *primary, unsigned long id); +int vmbus_get_device_info_os(struct rte_vmbus_device *dev); + +const char *get_devname_os(void); + #endif /* _VMBUS_BUS_OSI_H_ */ diff --git a/drivers/bus/vmbus/osi/vmbus_osi_uio.c b/drivers/bus/vmbus/osi/vmbus_osi_uio.c index 35106e247e..3374813cc8 100644 --- a/drivers/bus/vmbus/osi/vmbus_osi_uio.c +++ b/drivers/bus/vmbus/osi/vmbus_osi_uio.c @@ -82,7 +82,8 @@ vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, int fd; /* save fd if in primary process */ - snprintf(devname, sizeof(devname), "/dev/uio%u", dev->uio_num); + snprintf(devname, sizeof(devname), "%s%u", get_devname_os(), + dev->uio_num); fd = open(devname, O_RDWR); if (fd < 0) { VMBUS_LOG(ERR, "Cannot open %s: %s", @@ -106,6 +107,9 @@ vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, strlcpy((*uio_res)->path, devname, PATH_MAX); rte_uuid_copy((*uio_res)->id, dev->device_id); + if (vmbus_get_device_info_os(dev) < 0) + goto error; + return 0; error: -- 2.30.2
- Event monitoring is not yet supported on FreeBSD, hence moving it to the OS specific files - Add meson support to OS environment Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/freebsd/hn_os.c | 16 ++++++++++++++++ drivers/net/netvsc/freebsd/meson.build | 6 ++++++ drivers/net/netvsc/hn_ethdev.c | 7 +++---- drivers/net/netvsc/hn_os.h | 6 ++++++ drivers/net/netvsc/linux/hn_os.c | 21 +++++++++++++++++++++ drivers/net/netvsc/linux/meson.build | 6 ++++++ drivers/net/netvsc/meson.build | 3 +++ 7 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 drivers/net/netvsc/freebsd/hn_os.c create mode 100644 drivers/net/netvsc/freebsd/meson.build create mode 100644 drivers/net/netvsc/hn_os.h create mode 100644 drivers/net/netvsc/linux/hn_os.c create mode 100644 drivers/net/netvsc/linux/meson.build diff --git a/drivers/net/netvsc/freebsd/hn_os.c b/drivers/net/netvsc/freebsd/hn_os.c new file mode 100644 index 0000000000..4c6a79872d --- /dev/null +++ b/drivers/net/netvsc/freebsd/hn_os.c @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2021 Microsoft Corporation + */ + +#include <stdio.h> + +#include <rte_common.h> + +#include "hn_logs.h" +#include "hn_os.h" + +int eth_hn_os_dev_event(void) +{ + PMD_DRV_LOG(DEBUG, "rte_dev_event_monitor_start not supported on FreeBSD"); + return 0; +} diff --git a/drivers/net/netvsc/freebsd/meson.build b/drivers/net/netvsc/freebsd/meson.build new file mode 100644 index 0000000000..78f824f701 --- /dev/null +++ b/drivers/net/netvsc/freebsd/meson.build @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Microsoft Corporation + +sources += files( + 'hn_os.c', +) diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c index 8a950403ac..8b1e07b775 100644 --- a/drivers/net/netvsc/hn_ethdev.c +++ b/drivers/net/netvsc/hn_ethdev.c @@ -39,6 +39,7 @@ #include "hn_rndis.h" #include "hn_nvs.h" #include "ndis.h" +#include "hn_os.h" #define HN_TX_OFFLOAD_CAPS (RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | \ RTE_ETH_TX_OFFLOAD_TCP_CKSUM | \ @@ -1240,11 +1241,9 @@ static int eth_hn_probe(struct rte_vmbus_driver *drv __rte_unused, PMD_INIT_FUNC_TRACE(); - ret = rte_dev_event_monitor_start(); - if (ret) { - PMD_DRV_LOG(ERR, "Failed to start device event monitoring"); + ret = eth_hn_os_dev_event(); + if (ret) return ret; - } eth_dev = eth_dev_vmbus_allocate(dev, sizeof(struct hn_data)); if (!eth_dev) diff --git a/drivers/net/netvsc/hn_os.h b/drivers/net/netvsc/hn_os.h new file mode 100644 index 0000000000..618c53cdcd --- /dev/null +++ b/drivers/net/netvsc/hn_os.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2009-2021 Microsoft Corp. + * All rights reserved. + */ + +int eth_hn_os_dev_event(void); diff --git a/drivers/net/netvsc/linux/hn_os.c b/drivers/net/netvsc/linux/hn_os.c new file mode 100644 index 0000000000..1ea12ce928 --- /dev/null +++ b/drivers/net/netvsc/linux/hn_os.c @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2021 Microsoft Corporation + */ + +#include <unistd.h> + +#include <rte_ethdev.h> + +#include "hn_logs.h" +#include "hn_os.h" + +int eth_hn_os_dev_event(void) +{ + int ret; + + ret = rte_dev_event_monitor_start(); + if (ret) + PMD_DRV_LOG(ERR, "Failed to start device event monitoring"); + + return ret; +} diff --git a/drivers/net/netvsc/linux/meson.build b/drivers/net/netvsc/linux/meson.build new file mode 100644 index 0000000000..78f824f701 --- /dev/null +++ b/drivers/net/netvsc/linux/meson.build @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Microsoft Corporation + +sources += files( + 'hn_os.c', +) diff --git a/drivers/net/netvsc/meson.build b/drivers/net/netvsc/meson.build index bb6225d05a..c50414de07 100644 --- a/drivers/net/netvsc/meson.build +++ b/drivers/net/netvsc/meson.build @@ -8,6 +8,7 @@ if is_windows endif deps += 'bus_vmbus' +includes += include_directories(exec_env) sources = files( 'hn_ethdev.c', 'hn_nvs.c', @@ -15,3 +16,5 @@ sources = files( 'hn_rxtx.c', 'hn_vf.c', ) + +subdir(exec_env) -- 2.30.2
To map the subchannels, an mmap request is directly made after determining the subchan memory offset Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 48 +++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index b622388ce9..6b43b4349f 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -18,6 +18,13 @@ #include "private.h" #include "vmbus_osi.h" +/* + * Macros to distinguish mmap request + * [7-0] - Device memory region + * [15-8]- Sub-channel id + */ +#define UH_SUBCHAN_MASK_SHIFT 8 + const char *driver_name = "hv_uio"; /* Check map names with kernel names */ @@ -99,6 +106,47 @@ int vmbus_get_device_info_os(struct rte_vmbus_device *dev) return sysctl_get_vmbus_device_info(dev); } +int vmbus_uio_map_subchan_os(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan, + void **mapaddr, size_t *size) +{ + char ring_path[PATH_MAX]; + off_t offset; + int fd; + + snprintf(ring_path, sizeof(ring_path), + "/dev/hv_uio%d", dev->uio_num); + + fd = open(ring_path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + ring_path, strerror(errno)); + return -errno; + } + + /* subchannel rings are of the same size as primary */ + *size = dev->resource[HV_TXRX_RING_MAP].len; + offset = (chan->relid << UH_SUBCHAN_MASK_SHIFT) * PAGE_SIZE; + + *mapaddr = vmbus_map_resource(vmbus_map_addr, fd, + offset, *size, 0); + close(fd); + + if (*mapaddr == MAP_FAILED) + return -EIO; + + return 0; +} + +/* This function should always succeed */ +bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan) +{ + RTE_SET_USED(dev); + RTE_SET_USED(chan); + return true; +} + const char *get_devname_os(void) { return "/dev/hv_uio"; -- 2.30.2
In FreeBSD, unlike Linux there is no sub-channel open callback that could be called by HV_UIO driver upon their grant by the hypervisor. Thus the PMD makes an IOCTL to the HV_UIO to open the sub-channels On Linux, the vmbus_uio_subchan_open() will always return success as the Linux HV_UIO opens them implicitly. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 30 +++++++++++++++++++++++++++ drivers/bus/vmbus/linux/vmbus_uio.c | 12 +++++++++++ drivers/bus/vmbus/private.h | 1 + drivers/bus/vmbus/rte_bus_vmbus.h | 10 +++++++++ drivers/bus/vmbus/version.map | 1 + drivers/bus/vmbus/vmbus_channel.c | 5 +++++ 6 files changed, 59 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index 6b43b4349f..7e4e9723cb 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -25,6 +25,9 @@ */ #define UH_SUBCHAN_MASK_SHIFT 8 +/* ioctl */ +#define HVIOOPENSUBCHAN _IOW('h', 14, uint32_t) + const char *driver_name = "hv_uio"; /* Check map names with kernel names */ @@ -151,3 +154,30 @@ const char *get_devname_os(void) { return "/dev/hv_uio"; } + +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, uint32_t subchan) +{ + struct mapped_vmbus_resource *uio_res; + int fd, err = 0; + + uio_res = vmbus_uio_find_resource(dev); + if (!uio_res) { + VMBUS_LOG(ERR, "cannot find uio resource"); + return -EINVAL; + } + + fd = open(uio_res->path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + uio_res->path, strerror(errno)); + return -1; + } + + if (ioctl(fd, HVIOOPENSUBCHAN, &subchan)) { + VMBUS_LOG(ERR, "open subchan ioctl failed %s: %s", + uio_res->path, strerror(errno)); + err = -1; + } + close(fd); + return err; +} diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c b/drivers/bus/vmbus/linux/vmbus_uio.c index 669551a4d4..d88cc97b5b 100644 --- a/drivers/bus/vmbus/linux/vmbus_uio.c +++ b/drivers/bus/vmbus/linux/vmbus_uio.c @@ -215,3 +215,15 @@ const char *get_devname_os(void) { return "/dev/uio"; } + +/* + * This is a stub function and it should always succeed. + * The Linux UIO kernel driver opens the subchannels implicitly. + */ +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, + uint32_t subchan) +{ + RTE_SET_USED(dev); + RTE_SET_USED(subchan); + return 0; +} diff --git a/drivers/bus/vmbus/private.h b/drivers/bus/vmbus/private.h index 1bca147e12..ea0276a6c6 100644 --- a/drivers/bus/vmbus/private.h +++ b/drivers/bus/vmbus/private.h @@ -116,6 +116,7 @@ bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, int vmbus_uio_get_subchan(struct vmbus_channel *primary, struct vmbus_channel **subchan); int vmbus_uio_map_rings(struct vmbus_channel *chan); +int vmbus_uio_subchan_open(struct rte_vmbus_device *device, uint32_t subchan); void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen); diff --git a/drivers/bus/vmbus/rte_bus_vmbus.h b/drivers/bus/vmbus/rte_bus_vmbus.h index a24bad831d..c8cb4bc98c 100644 --- a/drivers/bus/vmbus/rte_bus_vmbus.h +++ b/drivers/bus/vmbus/rte_bus_vmbus.h @@ -404,6 +404,16 @@ void rte_vmbus_chan_dump(FILE *f, const struct vmbus_channel *chan); */ void rte_vmbus_unregister(struct rte_vmbus_driver *driver); +/** + * Perform IOCTL to VMBUS device + * + * @param device + * A pointer to a rte_vmbus_device structure + * @param subchan + * Count of subchannels to open + */ +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan); + /** Helper for VMBUS device registration from driver instance */ #define RTE_PMD_REGISTER_VMBUS(nm, vmbus_drv) \ RTE_INIT(vmbusinitfn_ ##nm) \ diff --git a/drivers/bus/vmbus/version.map b/drivers/bus/vmbus/version.map index 3cadec7fae..3509d4fc14 100644 --- a/drivers/bus/vmbus/version.map +++ b/drivers/bus/vmbus/version.map @@ -23,6 +23,7 @@ DPDK_22 { rte_vmbus_subchan_open; rte_vmbus_unmap_device; rte_vmbus_unregister; + rte_vmbus_ioctl; local: *; }; diff --git a/drivers/bus/vmbus/vmbus_channel.c b/drivers/bus/vmbus/vmbus_channel.c index 119b9b367e..9a8f6e3eef 100644 --- a/drivers/bus/vmbus/vmbus_channel.c +++ b/drivers/bus/vmbus/vmbus_channel.c @@ -365,6 +365,11 @@ int rte_vmbus_max_channels(const struct rte_vmbus_device *device) return 1; } +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan) +{ + return vmbus_uio_subchan_open(device, subchan); +} + /* Setup secondary channel */ int rte_vmbus_subchan_open(struct vmbus_channel *primary, struct vmbus_channel **new_chan) -- 2.30.2
make IOCTL call to open subchannels Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/hn_ethdev.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c index 8b1e07b775..104c7aebc5 100644 --- a/drivers/net/netvsc/hn_ethdev.c +++ b/drivers/net/netvsc/hn_ethdev.c @@ -516,6 +516,10 @@ static int hn_subchan_configure(struct hn_data *hv, if (err) return err; + err = rte_vmbus_ioctl(hv->vmbus, subchan); + if (err) + return err; + while (subchan > 0) { struct vmbus_channel *new_sc; uint16_t chn_index; -- 2.30.2
Using sysctl, all the subchannel's attributes are fetched Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 73 +++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index 7e4e9723cb..ecd62b31ca 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -155,6 +155,79 @@ const char *get_devname_os(void) return "/dev/hv_uio"; } +int vmbus_uio_get_subchan(struct vmbus_channel *primary, + struct vmbus_channel **subchan) +{ + const struct rte_vmbus_device *dev = primary->device; + char sysctlBuffer[PATH_MAX], sysctlVar[PATH_MAX]; + size_t len = PATH_MAX, sysctl_len; + /* nr_schan, relid, subid & monid datatype must match kernel's for sysctl */ + uint32_t relid, subid, nr_schan, i; + uint8_t monid; + int err; + + /* get no. of sub-channels opened by hv_uio + * dev.hv_uio.0.subchan_cnt + */ + snprintf(sysctlVar, len, "dev.%s.%d.subchan_cnt", driver_name, + dev->uio_num); + sysctl_len = sizeof(nr_schan); + if (sysctlbyname(sysctlVar, &nr_schan, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctlVar, + strerror(errno)); + return -1; + } + + /* dev.hv_uio.0.channel.14.sub */ + snprintf(sysctlBuffer, len, "dev.%s.%d.channel.%u.sub", driver_name, + dev->uio_num, primary->relid); + for (i = 1; i <= nr_schan; i++) { + /* get relid */ + snprintf(sysctlVar, len, "%s.%u.chanid", sysctlBuffer, i); + sysctl_len = sizeof(relid); + if (sysctlbyname(sysctlVar, &relid, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctlVar, + strerror(errno)); + goto error; + } + + if (!vmbus_isnew_subchannel(primary, (uint16_t)relid)) { + VMBUS_LOG(DEBUG, "skip already found channel: %u", + relid); + continue; + } + + /* get sub-channel id */ + snprintf(sysctlVar, len, "%s.%u.ch_subidx", sysctlBuffer, i); + sysctl_len = sizeof(subid); + if (sysctlbyname(sysctlVar, &subid, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctlVar, + strerror(errno)); + goto error; + } + + /* get monitor id */ + snprintf(sysctlVar, len, "%s.%u.monitor_id", sysctlBuffer, i); + sysctl_len = sizeof(monid); + if (sysctlbyname(sysctlVar, &monid, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctlVar, + strerror(errno)); + goto error; + } + + err = vmbus_chan_create(dev, (uint16_t)relid, (uint16_t)subid, + monid, subchan); + if (err) { + VMBUS_LOG(ERR, "subchannel setup failed"); + return err; + } + break; + } + return 0; +error: + return -1; +} + int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, uint32_t subchan) { struct mapped_vmbus_resource *uio_res; -- 2.30.2
Moved netvsc_hotplug_retry to respective OS dir as it contains OS dependent code. For Linux, it is copied as is and for FreeBSD it is not supported yet. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/freebsd/hn_os.c | 5 ++ drivers/net/netvsc/hn_ethdev.c | 84 ---------------------------- drivers/net/netvsc/hn_os.h | 2 + drivers/net/netvsc/linux/hn_os.c | 90 ++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 84 deletions(-) diff --git a/drivers/net/netvsc/freebsd/hn_os.c b/drivers/net/netvsc/freebsd/hn_os.c index 4c6a79872d..fece1be9c9 100644 --- a/drivers/net/netvsc/freebsd/hn_os.c +++ b/drivers/net/netvsc/freebsd/hn_os.c @@ -14,3 +14,8 @@ int eth_hn_os_dev_event(void) PMD_DRV_LOG(DEBUG, "rte_dev_event_monitor_start not supported on FreeBSD"); return 0; } + +void netvsc_hotplug_retry(void *args) +{ + RTE_SET_USED(args); +} diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c index 104c7aebc5..dd4b872fed 100644 --- a/drivers/net/netvsc/hn_ethdev.c +++ b/drivers/net/netvsc/hn_ethdev.c @@ -57,9 +57,6 @@ #define NETVSC_ARG_TXBREAK "tx_copybreak" #define NETVSC_ARG_RX_EXTMBUF_ENABLE "rx_extmbuf_enable" -/* The max number of retry when hot adding a VF device */ -#define NETVSC_MAX_HOTADD_RETRY 10 - struct hn_xstats_name_off { char name[RTE_ETH_XSTATS_NAME_SIZE]; unsigned int offset; @@ -556,87 +553,6 @@ static int hn_subchan_configure(struct hn_data *hv, return err; } -static void netvsc_hotplug_retry(void *args) -{ - int ret; - struct hn_data *hv = args; - struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id]; - struct rte_devargs *d = &hv->devargs; - char buf[256]; - - DIR *di; - struct dirent *dir; - struct ifreq req; - struct rte_ether_addr eth_addr; - int s; - - PMD_DRV_LOG(DEBUG, "%s: retry count %d", - __func__, hv->eal_hot_plug_retry); - - if (hv->eal_hot_plug_retry++ > NETVSC_MAX_HOTADD_RETRY) - return; - - snprintf(buf, sizeof(buf), "/sys/bus/pci/devices/%s/net", d->name); - di = opendir(buf); - if (!di) { - PMD_DRV_LOG(DEBUG, "%s: can't open directory %s, " - "retrying in 1 second", __func__, buf); - goto retry; - } - - while ((dir = readdir(di))) { - /* Skip . and .. directories */ - if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) - continue; - - /* trying to get mac address if this is a network device*/ - s = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); - if (s == -1) { - PMD_DRV_LOG(ERR, "Failed to create socket errno %d", - errno); - break; - } - strlcpy(req.ifr_name, dir->d_name, sizeof(req.ifr_name)); - ret = ioctl(s, SIOCGIFHWADDR, &req); - close(s); - if (ret == -1) { - PMD_DRV_LOG(ERR, - "Failed to send SIOCGIFHWADDR for device %s", - dir->d_name); - break; - } - if (req.ifr_hwaddr.sa_family != ARPHRD_ETHER) { - closedir(di); - return; - } - memcpy(eth_addr.addr_bytes, req.ifr_hwaddr.sa_data, - RTE_DIM(eth_addr.addr_bytes)); - - if (rte_is_same_ether_addr(ð_addr, dev->data->mac_addrs)) { - PMD_DRV_LOG(NOTICE, - "Found matching MAC address, adding device %s network name %s", - d->name, dir->d_name); - ret = rte_eal_hotplug_add(d->bus->name, d->name, - d->args); - if (ret) { - PMD_DRV_LOG(ERR, - "Failed to add PCI device %s", - d->name); - break; - } - } - /* When the code reaches here, we either have already added - * the device, or its MAC address did not match. - */ - closedir(di); - return; - } - closedir(di); -retry: - /* The device is still being initialized, retry after 1 second */ - rte_eal_alarm_set(1000000, netvsc_hotplug_retry, hv); -} - static void netvsc_hotadd_callback(const char *device_name, enum rte_dev_event_type type, void *arg) diff --git a/drivers/net/netvsc/hn_os.h b/drivers/net/netvsc/hn_os.h index 618c53cdcd..1fb7292b17 100644 --- a/drivers/net/netvsc/hn_os.h +++ b/drivers/net/netvsc/hn_os.h @@ -4,3 +4,5 @@ */ int eth_hn_os_dev_event(void); + +void netvsc_hotplug_retry(void *args); diff --git a/drivers/net/netvsc/linux/hn_os.c b/drivers/net/netvsc/linux/hn_os.c index 1ea12ce928..9c2f4cd7a8 100644 --- a/drivers/net/netvsc/linux/hn_os.c +++ b/drivers/net/netvsc/linux/hn_os.c @@ -3,12 +3,21 @@ */ #include <unistd.h> +#include <dirent.h> +#include <net/if.h> +#include <net/if_arp.h> +#include <sys/ioctl.h> #include <rte_ethdev.h> +#include <rte_alarm.h> #include "hn_logs.h" +#include "hn_var.h" #include "hn_os.h" +/* The max number of retry when hot adding a VF device */ +#define NETVSC_MAX_HOTADD_RETRY 10 + int eth_hn_os_dev_event(void) { int ret; @@ -19,3 +28,84 @@ int eth_hn_os_dev_event(void) return ret; } + +void netvsc_hotplug_retry(void *args) +{ + int ret; + struct hn_data *hv = args; + struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id]; + struct rte_devargs *d = &hv->devargs; + char buf[256]; + + DIR *di; + struct dirent *dir; + struct ifreq req; + struct rte_ether_addr eth_addr; + int s; + + PMD_DRV_LOG(DEBUG, "%s: retry count %d", + __func__, hv->eal_hot_plug_retry); + + if (hv->eal_hot_plug_retry++ > NETVSC_MAX_HOTADD_RETRY) + return; + + snprintf(buf, sizeof(buf), "/sys/bus/pci/devices/%s/net", d->name); + di = opendir(buf); + if (!di) { + PMD_DRV_LOG(DEBUG, "%s: can't open directory %s, " + "retrying in 1 second", __func__, buf); + goto retry; + } + + while ((dir = readdir(di))) { + /* Skip . and .. directories */ + if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) + continue; + + /* trying to get mac address if this is a network device*/ + s = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); + if (s == -1) { + PMD_DRV_LOG(ERR, "Failed to create socket errno %d", + errno); + break; + } + strlcpy(req.ifr_name, dir->d_name, sizeof(req.ifr_name)); + ret = ioctl(s, SIOCGIFHWADDR, &req); + close(s); + if (ret == -1) { + PMD_DRV_LOG(ERR, + "Failed to send SIOCGIFHWADDR for device %s", + dir->d_name); + break; + } + if (req.ifr_hwaddr.sa_family != ARPHRD_ETHER) { + closedir(di); + return; + } + memcpy(eth_addr.addr_bytes, req.ifr_hwaddr.sa_data, + RTE_DIM(eth_addr.addr_bytes)); + + if (rte_is_same_ether_addr(ð_addr, dev->data->mac_addrs)) { + PMD_DRV_LOG(NOTICE, + "Found matching MAC address, adding device %s network name %s", + d->name, dir->d_name); + ret = rte_eal_hotplug_add(d->bus->name, d->name, + d->args); + if (ret) { + PMD_DRV_LOG(ERR, + "Failed to add PCI device %s", + d->name); + break; + } + } + /* When the code reaches here, we either have already added + * the device, or its MAC address did not match. + */ + closedir(di); + return; + } + closedir(di); +retry: + /* The device is still being initialized, retry after 1 second */ + rte_eal_alarm_set(1000000, netvsc_hotplug_retry, hv); +} -- 2.30.2
add meson support for FreeBSD OS Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/meson.build | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build index fe9c72bce1..84ad434177 100644 --- a/drivers/bus/vmbus/meson.build +++ b/drivers/bus/vmbus/meson.build @@ -26,7 +26,11 @@ if is_linux sources += files('linux/vmbus_bus.c', 'linux/vmbus_uio.c') includes += include_directories('linux') +elif is_freebsd + sources += files('freebsd/vmbus_bus.c', + 'freebsd/vmbus_uio.c') + includes += include_directories('freebsd') else build = false - reason = 'only supported on Linux' + reason = 'only supported on Linux & FreeBSD' endif -- 2.30.2
updated MAINTAINERS and doc files for FreeBSD support Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- MAINTAINERS | 2 ++ doc/guides/nics/netvsc.rst | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 7c4f541dba..01a494ef0e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -567,6 +567,7 @@ F: app/test/test_vdev.c VMBUS bus driver M: Stephen Hemminger <sthemmin@microsoft.com> M: Long Li <longli@microsoft.com> +M: Srikanth Kaka <srikanth.k@oneconvergence.com> F: drivers/bus/vmbus/ @@ -823,6 +824,7 @@ F: doc/guides/nics/vdev_netvsc.rst Microsoft Hyper-V netvsc M: Stephen Hemminger <sthemmin@microsoft.com> M: Long Li <longli@microsoft.com> +M: Srikanth Kaka <srikanth.k@oneconvergence.com> F: drivers/net/netvsc/ F: doc/guides/nics/netvsc.rst F: doc/guides/nics/features/netvsc.ini diff --git a/doc/guides/nics/netvsc.rst b/doc/guides/nics/netvsc.rst index 77efe1dc91..12d17024b7 100644 --- a/doc/guides/nics/netvsc.rst +++ b/doc/guides/nics/netvsc.rst @@ -91,6 +91,12 @@ operations: The dpdk-devbind.py script can not be used since it only handles PCI devices. +On FreeBSD, with hv_uio kernel driver loaded, do the following: + + .. code-block:: console + + devctl set driver -f hn1 hv_uio + Prerequisites ------------- @@ -101,6 +107,11 @@ The following prerequisites apply: Full support of multiple queues requires the 4.17 kernel. It is possible to use the netvsc PMD with 4.16 kernel but it is limited to a single queue. +* FreeBSD support for UIO on vmbus is done with hv_uio driver and it is still + in `review`_ + +.. _`review`: https://reviews.freebsd.org/D32184 + Netvsc PMD arguments -------------------- -- 2.30.2
On Mon, 18 Apr 2022 09:59:02 +0530
Srikanth Kaka <srikanth.k@oneconvergence.com> wrote:
> Move the OS independent code from Linux dir in-order to be used
> by FreeBSD
>
> Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com>
> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com>
> Signed-off-by: Anand Thulasiram <avelu@juniper.net>
> ---
> drivers/bus/vmbus/linux/vmbus_bus.c | 13 +------------
> drivers/bus/vmbus/meson.build | 5 +++++
> drivers/bus/vmbus/osi/vmbus_osi.h | 11 +++++++++++
> drivers/bus/vmbus/osi/vmbus_osi_bus.c | 20 ++++++++++++++++++++
> 4 files changed, 37 insertions(+), 12 deletions(-)
> create mode 100644 drivers/bus/vmbus/osi/vmbus_osi.h
> create mode 100644 drivers/bus/vmbus/osi/vmbus_osi_bus.c
>
> diff --git a/drivers/bus/vmbus/linux/vmbus_bus.c b/drivers/bus/vmbus/linux/vmbus_bus.c
> index f502783f7a..c9a07041a7 100644
> --- a/drivers/bus/vmbus/linux/vmbus_bus.c
> +++ b/drivers/bus/vmbus/linux/vmbus_bus.c
> @@ -21,22 +21,11 @@
>
> #include "eal_filesystem.h"
> #include "private.h"
> +#include "vmbus_osi.h"
>
> /** Pathname of VMBUS devices directory. */
> #define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices"
>
> -/*
> - * GUID associated with network devices
> - * {f8615163-df3e-46c5-913f-f2d2f965ed0e}
> - */
> -static const rte_uuid_t vmbus_nic_uuid = {
> - 0xf8, 0x61, 0x51, 0x63,
> - 0xdf, 0x3e,
> - 0x46, 0xc5,
> - 0x91, 0x3f,
> - 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0xe
> -};
> -
> extern struct rte_vmbus_bus rte_vmbus_bus;
>
> /* Read sysfs file to get UUID */
> diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build
> index 3892cbf67f..cbcba44e16 100644
> --- a/drivers/bus/vmbus/meson.build
> +++ b/drivers/bus/vmbus/meson.build
> @@ -16,6 +16,11 @@ sources = files(
> 'vmbus_common_uio.c',
> )
>
> +includes += include_directories('osi')
> +sources += files(
> + 'osi/vmbus_osi_bus.c'
> +)
> +
> if is_linux
> sources += files('linux/vmbus_bus.c',
> 'linux/vmbus_uio.c')
> diff --git a/drivers/bus/vmbus/osi/vmbus_osi.h b/drivers/bus/vmbus/osi/vmbus_osi.h
> new file mode 100644
> index 0000000000..2db9399181
> --- /dev/null
> +++ b/drivers/bus/vmbus/osi/vmbus_osi.h
Having common code is good, we are already doing it now in DPDK EAL.
But the name osi seems odd to me.
Could you use unix instead (same as EAL)
drivers/bus/vmbus/unix/vmbus.h
Or drivers/bus/vmbus/common
On Mon, 18 Apr 2022 09:59:12 +0530
Srikanth Kaka <srikanth.k@oneconvergence.com> wrote:
> ice;
> + char sysctlBuffer[PATH_MAX], sysctlVar[PATH_MAX];
nit. please avoid using camelCase variable names.
[-- Attachment #1: Type: text/plain, Size: 2716 bytes --] Sure Stephen. I will change it to unix. On Tue, 19 Apr 2022, 8:19 pm Stephen Hemminger, <stephen@networkplumber.org> wrote: > On Mon, 18 Apr 2022 09:59:02 +0530 > Srikanth Kaka <srikanth.k@oneconvergence.com> wrote: > > > Move the OS independent code from Linux dir in-order to be used > > by FreeBSD > > > > Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> > > Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> > > Signed-off-by: Anand Thulasiram <avelu@juniper.net> > > --- > > drivers/bus/vmbus/linux/vmbus_bus.c | 13 +------------ > > drivers/bus/vmbus/meson.build | 5 +++++ > > drivers/bus/vmbus/osi/vmbus_osi.h | 11 +++++++++++ > > drivers/bus/vmbus/osi/vmbus_osi_bus.c | 20 ++++++++++++++++++++ > > 4 files changed, 37 insertions(+), 12 deletions(-) > > create mode 100644 drivers/bus/vmbus/osi/vmbus_osi.h > > create mode 100644 drivers/bus/vmbus/osi/vmbus_osi_bus.c > > > > diff --git a/drivers/bus/vmbus/linux/vmbus_bus.c > b/drivers/bus/vmbus/linux/vmbus_bus.c > > index f502783f7a..c9a07041a7 100644 > > --- a/drivers/bus/vmbus/linux/vmbus_bus.c > > +++ b/drivers/bus/vmbus/linux/vmbus_bus.c > > @@ -21,22 +21,11 @@ > > > > #include "eal_filesystem.h" > > #include "private.h" > > +#include "vmbus_osi.h" > > > > /** Pathname of VMBUS devices directory. */ > > #define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" > > > > -/* > > - * GUID associated with network devices > > - * {f8615163-df3e-46c5-913f-f2d2f965ed0e} > > - */ > > -static const rte_uuid_t vmbus_nic_uuid = { > > - 0xf8, 0x61, 0x51, 0x63, > > - 0xdf, 0x3e, > > - 0x46, 0xc5, > > - 0x91, 0x3f, > > - 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0xe > > -}; > > - > > extern struct rte_vmbus_bus rte_vmbus_bus; > > > > /* Read sysfs file to get UUID */ > > diff --git a/drivers/bus/vmbus/meson.build > b/drivers/bus/vmbus/meson.build > > index 3892cbf67f..cbcba44e16 100644 > > --- a/drivers/bus/vmbus/meson.build > > +++ b/drivers/bus/vmbus/meson.build > > @@ -16,6 +16,11 @@ sources = files( > > 'vmbus_common_uio.c', > > ) > > > > +includes += include_directories('osi') > > +sources += files( > > + 'osi/vmbus_osi_bus.c' > > +) > > + > > if is_linux > > sources += files('linux/vmbus_bus.c', > > 'linux/vmbus_uio.c') > > diff --git a/drivers/bus/vmbus/osi/vmbus_osi.h > b/drivers/bus/vmbus/osi/vmbus_osi.h > > new file mode 100644 > > index 0000000000..2db9399181 > > --- /dev/null > > +++ b/drivers/bus/vmbus/osi/vmbus_osi.h > > Having common code is good, we are already doing it now in DPDK EAL. > But the name osi seems odd to me. > Could you use unix instead (same as EAL) > > drivers/bus/vmbus/unix/vmbus.h > > Or drivers/bus/vmbus/common > [-- Attachment #2: Type: text/html, Size: 3902 bytes --]
On Mon, 18 Apr 2022 09:59:10 +0530
Srikanth Kaka <srikanth.k@oneconvergence.com> wrote:
> diff --git a/drivers/bus/vmbus/version.map b/drivers/bus/vmbus/version.map
> index 3cadec7fae..3509d4fc14 100644
> --- a/drivers/bus/vmbus/version.map
> +++ b/drivers/bus/vmbus/version.map
> @@ -23,6 +23,7 @@ DPDK_22 {
> rte_vmbus_subchan_open;
> rte_vmbus_unmap_device;
> rte_vmbus_unregister;
> + rte_vmbus_ioctl;
>
> local: *;
> };
Adding ioctl is fine, probably would have done on Linux if the kernel
community would have allowed it...
But it needs to go in EXPERIMENTAL section per DPDK policy.
This patchset requires FreeBSD VMBus kernel changes and HV_UIO driver. Both are currently under review at https://reviews.freebsd.org/D32184 Changelog: v5: - renamed dir osi to unix - marked a newly added API as experimental - removed camel case variables v4: - moved OS independent code out of Linux v3: - split the patches into further logical parts - updated docs v2: - replaced strncpy with memcpy - replaced malloc.h with stdlib.h - added comment in linux/vmbus_uio.c v1: Intial release Srikanth Kaka (14): bus/vmbus: move independent code from Linux bus/vmbus: move independent bus functions bus/vmbus: move OS independent UIO functions bus/vmbus: scan and get the network device on FreeBSD bus/vmbus: handle mapping of device resources bus/vmbus: get device resource values using sysctl net/netvsc: make event monitor OS dependent bus/vmbus: add sub-channel mapping support bus/vmbus: open subchannels net/netvsc: make IOCTL call to open subchannels bus/vmbus: get subchannel info net/netvsc: moving hotplug retry to OS dir bus/vmbus: add meson support for FreeBSD bus/vmbus: update MAINTAINERS and docs MAINTAINERS | 2 + doc/guides/nics/netvsc.rst | 11 ++ drivers/bus/vmbus/freebsd/vmbus_bus.c | 286 ++++++++++++++++++++++++++++ drivers/bus/vmbus/freebsd/vmbus_uio.c | 256 +++++++++++++++++++++++++ drivers/bus/vmbus/linux/vmbus_bus.c | 28 +-- drivers/bus/vmbus/linux/vmbus_uio.c | 320 ++++---------------------------- drivers/bus/vmbus/meson.build | 12 +- drivers/bus/vmbus/private.h | 1 + drivers/bus/vmbus/rte_bus_vmbus.h | 11 ++ drivers/bus/vmbus/unix/vmbus_unix.h | 27 +++ drivers/bus/vmbus/unix/vmbus_unix_bus.c | 37 ++++ drivers/bus/vmbus/unix/vmbus_unix_uio.c | 310 +++++++++++++++++++++++++++++++ drivers/bus/vmbus/version.map | 6 + drivers/bus/vmbus/vmbus_channel.c | 5 + drivers/net/netvsc/freebsd/hn_os.c | 21 +++ drivers/net/netvsc/freebsd/meson.build | 6 + drivers/net/netvsc/hn_ethdev.c | 95 +--------- drivers/net/netvsc/hn_os.h | 8 + drivers/net/netvsc/linux/hn_os.c | 111 +++++++++++ drivers/net/netvsc/linux/meson.build | 6 + drivers/net/netvsc/meson.build | 3 + 21 files changed, 1164 insertions(+), 398 deletions(-) create mode 100644 drivers/bus/vmbus/freebsd/vmbus_bus.c create mode 100644 drivers/bus/vmbus/freebsd/vmbus_uio.c create mode 100644 drivers/bus/vmbus/unix/vmbus_unix.h create mode 100644 drivers/bus/vmbus/unix/vmbus_unix_bus.c create mode 100644 drivers/bus/vmbus/unix/vmbus_unix_uio.c create mode 100644 drivers/net/netvsc/freebsd/hn_os.c create mode 100644 drivers/net/netvsc/freebsd/meson.build create mode 100644 drivers/net/netvsc/hn_os.h create mode 100644 drivers/net/netvsc/linux/hn_os.c create mode 100644 drivers/net/netvsc/linux/meson.build -- 1.8.3.1
Move the OS independent code from Linux dir in-order to be used by FreeBSD Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/linux/vmbus_bus.c | 13 +------------ drivers/bus/vmbus/meson.build | 5 +++++ drivers/bus/vmbus/unix/vmbus_unix.h | 11 +++++++++++ drivers/bus/vmbus/unix/vmbus_unix_bus.c | 20 ++++++++++++++++++++ 4 files changed, 37 insertions(+), 12 deletions(-) create mode 100644 drivers/bus/vmbus/unix/vmbus_unix.h create mode 100644 drivers/bus/vmbus/unix/vmbus_unix_bus.c diff --git a/drivers/bus/vmbus/linux/vmbus_bus.c b/drivers/bus/vmbus/linux/vmbus_bus.c index f502783..e649537 100644 --- a/drivers/bus/vmbus/linux/vmbus_bus.c +++ b/drivers/bus/vmbus/linux/vmbus_bus.c @@ -21,22 +21,11 @@ #include "eal_filesystem.h" #include "private.h" +#include "vmbus_unix.h" /** Pathname of VMBUS devices directory. */ #define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" -/* - * GUID associated with network devices - * {f8615163-df3e-46c5-913f-f2d2f965ed0e} - */ -static const rte_uuid_t vmbus_nic_uuid = { - 0xf8, 0x61, 0x51, 0x63, - 0xdf, 0x3e, - 0x46, 0xc5, - 0x91, 0x3f, - 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0xe -}; - extern struct rte_vmbus_bus rte_vmbus_bus; /* Read sysfs file to get UUID */ diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build index 3892cbf..01ef01f 100644 --- a/drivers/bus/vmbus/meson.build +++ b/drivers/bus/vmbus/meson.build @@ -16,6 +16,11 @@ sources = files( 'vmbus_common_uio.c', ) +includes += include_directories('unix') +sources += files( + 'unix/vmbus_unix_bus.c' +) + if is_linux sources += files('linux/vmbus_bus.c', 'linux/vmbus_uio.c') diff --git a/drivers/bus/vmbus/unix/vmbus_unix.h b/drivers/bus/vmbus/unix/vmbus_unix.h new file mode 100644 index 0000000..2db9399 --- /dev/null +++ b/drivers/bus/vmbus/unix/vmbus_unix.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#ifndef _VMBUS_BUS_UNIX_H_ +#define _VMBUS_BUS_UNIX_H_ + +extern const rte_uuid_t vmbus_nic_uuid; + +#endif /* _VMBUS_BUS_UNIX_H_ */ diff --git a/drivers/bus/vmbus/unix/vmbus_unix_bus.c b/drivers/bus/vmbus/unix/vmbus_unix_bus.c new file mode 100644 index 0000000..f76a361 --- /dev/null +++ b/drivers/bus/vmbus/unix/vmbus_unix_bus.c @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <rte_uuid.h> + +#include "vmbus_unix.h" + +/* + * GUID associated with network devices + * {f8615163-df3e-46c5-913f-f2d2f965ed0e} + */ +const rte_uuid_t vmbus_nic_uuid = { + 0xf8, 0x61, 0x51, 0x63, + 0xdf, 0x3e, + 0x46, 0xc5, + 0x91, 0x3f, + 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0xe +}; -- 1.8.3.1
move independent Linux bus functions to OS independent file Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/linux/vmbus_bus.c | 15 --------------- drivers/bus/vmbus/unix/vmbus_unix_bus.c | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/drivers/bus/vmbus/linux/vmbus_bus.c b/drivers/bus/vmbus/linux/vmbus_bus.c index e649537..18233a5 100644 --- a/drivers/bus/vmbus/linux/vmbus_bus.c +++ b/drivers/bus/vmbus/linux/vmbus_bus.c @@ -358,18 +358,3 @@ closedir(dir); return -1; } - -void rte_vmbus_irq_mask(struct rte_vmbus_device *device) -{ - vmbus_uio_irq_control(device, 1); -} - -void rte_vmbus_irq_unmask(struct rte_vmbus_device *device) -{ - vmbus_uio_irq_control(device, 0); -} - -int rte_vmbus_irq_read(struct rte_vmbus_device *device) -{ - return vmbus_uio_irq_read(device); -} diff --git a/drivers/bus/vmbus/unix/vmbus_unix_bus.c b/drivers/bus/vmbus/unix/vmbus_unix_bus.c index f76a361..96cb968 100644 --- a/drivers/bus/vmbus/unix/vmbus_unix_bus.c +++ b/drivers/bus/vmbus/unix/vmbus_unix_bus.c @@ -3,8 +3,10 @@ * All Rights Reserved. */ +#include <rte_bus.h> #include <rte_uuid.h> +#include "private.h" #include "vmbus_unix.h" /* @@ -18,3 +20,18 @@ 0x91, 0x3f, 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0xe }; + +void rte_vmbus_irq_mask(struct rte_vmbus_device *device) +{ + vmbus_uio_irq_control(device, 1); +} + +void rte_vmbus_irq_unmask(struct rte_vmbus_device *device) +{ + vmbus_uio_irq_control(device, 0); +} + +int rte_vmbus_irq_read(struct rte_vmbus_device *device) +{ + return vmbus_uio_irq_read(device); +} -- 1.8.3.1
Moved all Linux independent UIO functions to unix dir. Split the vmbus_uio_map_subchan() by keeping OS dependent code in vmbus_uio_map_subchan_os() function Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/linux/vmbus_uio.c | 292 ++---------------------------- drivers/bus/vmbus/meson.build | 3 +- drivers/bus/vmbus/unix/vmbus_unix.h | 12 ++ drivers/bus/vmbus/unix/vmbus_unix_uio.c | 306 ++++++++++++++++++++++++++++++++ 4 files changed, 330 insertions(+), 283 deletions(-) create mode 100644 drivers/bus/vmbus/unix/vmbus_unix_uio.c diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c b/drivers/bus/vmbus/linux/vmbus_uio.c index 5db70f8..b5d15c9 100644 --- a/drivers/bus/vmbus/linux/vmbus_uio.c +++ b/drivers/bus/vmbus/linux/vmbus_uio.c @@ -21,233 +21,18 @@ #include <rte_string_fns.h> #include "private.h" +#include "vmbus_unix.h" /** Pathname of VMBUS devices directory. */ #define SYSFS_VMBUS_DEVICES "/sys/bus/vmbus/devices" -static void *vmbus_map_addr; - -/* Control interrupts */ -void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff) -{ - if ((rte_intr_fd_get(dev->intr_handle) < 0) || - write(rte_intr_fd_get(dev->intr_handle), &onoff, - sizeof(onoff)) < 0) { - VMBUS_LOG(ERR, "cannot write to %d:%s", - rte_intr_fd_get(dev->intr_handle), - strerror(errno)); - } -} - -int vmbus_uio_irq_read(struct rte_vmbus_device *dev) -{ - int32_t count; - int cc; - - if (rte_intr_fd_get(dev->intr_handle) < 0) - return -1; - - cc = read(rte_intr_fd_get(dev->intr_handle), &count, - sizeof(count)); - if (cc < (int)sizeof(count)) { - if (cc < 0) { - VMBUS_LOG(ERR, "IRQ read failed %s", - strerror(errno)); - return -errno; - } - VMBUS_LOG(ERR, "can't read IRQ count"); - return -EINVAL; - } - - return count; -} - -void -vmbus_uio_free_resource(struct rte_vmbus_device *dev, - struct mapped_vmbus_resource *uio_res) -{ - rte_free(uio_res); - - if (rte_intr_dev_fd_get(dev->intr_handle) >= 0) { - close(rte_intr_dev_fd_get(dev->intr_handle)); - rte_intr_dev_fd_set(dev->intr_handle, -1); - } - - if (rte_intr_fd_get(dev->intr_handle) >= 0) { - close(rte_intr_fd_get(dev->intr_handle)); - rte_intr_fd_set(dev->intr_handle, -1); - rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UNKNOWN); - } -} - -int -vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, - struct mapped_vmbus_resource **uio_res) -{ - char devname[PATH_MAX]; /* contains the /dev/uioX */ - int fd; - - /* save fd if in primary process */ - snprintf(devname, sizeof(devname), "/dev/uio%u", dev->uio_num); - fd = open(devname, O_RDWR); - if (fd < 0) { - VMBUS_LOG(ERR, "Cannot open %s: %s", - devname, strerror(errno)); - goto error; - } - - if (rte_intr_fd_set(dev->intr_handle, fd)) - goto error; - - if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UIO_INTX)) - goto error; - - /* allocate the mapping details for secondary processes*/ - *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); - if (*uio_res == NULL) { - VMBUS_LOG(ERR, "cannot store uio mmap details"); - goto error; - } - - strlcpy((*uio_res)->path, devname, PATH_MAX); - rte_uuid_copy((*uio_res)->id, dev->device_id); - - return 0; - -error: - vmbus_uio_free_resource(dev, *uio_res); - return -1; -} - -static int -find_max_end_va(const struct rte_memseg_list *msl, void *arg) -{ - size_t sz = msl->memseg_arr.len * msl->page_sz; - void *end_va = RTE_PTR_ADD(msl->base_va, sz); - void **max_va = arg; - - if (*max_va < end_va) - *max_va = end_va; - return 0; -} - -/* - * TODO: this should be part of memseg api. - * code is duplicated from PCI. - */ -static void * -vmbus_find_max_end_va(void) -{ - void *va = NULL; - - rte_memseg_list_walk(find_max_end_va, &va); - return va; -} - -int -vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int idx, - struct mapped_vmbus_resource *uio_res, - int flags) -{ - size_t size = dev->resource[idx].len; - struct vmbus_map *maps = uio_res->maps; - void *mapaddr; - off_t offset; - int fd; - - /* devname for mmap */ - fd = open(uio_res->path, O_RDWR); - if (fd < 0) { - VMBUS_LOG(ERR, "Cannot open %s: %s", - uio_res->path, strerror(errno)); - return -1; - } - - /* try mapping somewhere close to the end of hugepages */ - if (vmbus_map_addr == NULL) - vmbus_map_addr = vmbus_find_max_end_va(); - - /* offset is special in uio it indicates which resource */ - offset = idx * rte_mem_page_size(); - - mapaddr = vmbus_map_resource(vmbus_map_addr, fd, offset, size, flags); - close(fd); - - if (mapaddr == MAP_FAILED) - return -1; - - dev->resource[idx].addr = mapaddr; - vmbus_map_addr = RTE_PTR_ADD(mapaddr, size); - - /* Record result of successful mapping for use by secondary */ - maps[idx].addr = mapaddr; - maps[idx].size = size; - - return 0; -} - -static int vmbus_uio_map_primary(struct vmbus_channel *chan, - void **ring_buf, uint32_t *ring_size) -{ - struct mapped_vmbus_resource *uio_res; - - uio_res = vmbus_uio_find_resource(chan->device); - if (!uio_res) { - VMBUS_LOG(ERR, "can not find resources!"); - return -ENOMEM; - } - - if (uio_res->nb_maps < VMBUS_MAX_RESOURCE) { - VMBUS_LOG(ERR, "VMBUS: only %u resources found!", - uio_res->nb_maps); - return -EINVAL; - } - - *ring_size = uio_res->maps[HV_TXRX_RING_MAP].size / 2; - *ring_buf = uio_res->maps[HV_TXRX_RING_MAP].addr; - return 0; -} - -static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev, - const struct vmbus_channel *chan, - void **ring_buf, uint32_t *ring_size) +int vmbus_uio_map_subchan_os(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan, + void **mapaddr, size_t *file_size) { char ring_path[PATH_MAX]; - size_t file_size; struct stat sb; - void *mapaddr; int fd; - struct mapped_vmbus_resource *uio_res; - int channel_idx; - - uio_res = vmbus_uio_find_resource(dev); - if (!uio_res) { - VMBUS_LOG(ERR, "can not find resources for mapping subchan"); - return -ENOMEM; - } - - if (rte_eal_process_type() == RTE_PROC_PRIMARY) { - if (uio_res->nb_subchannels >= UIO_MAX_SUBCHANNEL) { - VMBUS_LOG(ERR, - "exceeding max subchannels UIO_MAX_SUBCHANNEL(%d)", - UIO_MAX_SUBCHANNEL); - VMBUS_LOG(ERR, "Change UIO_MAX_SUBCHANNEL and recompile"); - return -ENOMEM; - } - } else { - for (channel_idx = 0; channel_idx < uio_res->nb_subchannels; - channel_idx++) - if (uio_res->subchannel_maps[channel_idx].relid == - chan->relid) - break; - if (channel_idx == uio_res->nb_subchannels) { - VMBUS_LOG(ERR, - "couldn't find sub channel %d from shared mapping in primary", - chan->relid); - return -ENOMEM; - } - vmbus_map_addr = uio_res->subchannel_maps[channel_idx].addr; - } snprintf(ring_path, sizeof(ring_path), "%s/%s/channels/%u/ring", @@ -267,68 +52,23 @@ static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev, close(fd); return -errno; } - file_size = sb.st_size; + *file_size = sb.st_size; - if (file_size == 0 || (file_size & (rte_mem_page_size() - 1))) { + if (*file_size == 0 || (*file_size & (rte_mem_page_size() - 1))) { VMBUS_LOG(ERR, "incorrect size %s: %zu", - ring_path, file_size); + ring_path, *file_size); close(fd); return -EINVAL; } - mapaddr = vmbus_map_resource(vmbus_map_addr, fd, - 0, file_size, 0); + *mapaddr = vmbus_map_resource(vmbus_map_addr, fd, + 0, *file_size, 0); close(fd); - if (mapaddr == MAP_FAILED) + if (*mapaddr == MAP_FAILED) return -EIO; - if (rte_eal_process_type() == RTE_PROC_PRIMARY) { - - /* Add this mapping to uio_res for use by secondary */ - uio_res->subchannel_maps[uio_res->nb_subchannels].relid = - chan->relid; - uio_res->subchannel_maps[uio_res->nb_subchannels].addr = - mapaddr; - uio_res->subchannel_maps[uio_res->nb_subchannels].size = - file_size; - uio_res->nb_subchannels++; - - vmbus_map_addr = RTE_PTR_ADD(mapaddr, file_size); - } else { - if (mapaddr != vmbus_map_addr) { - VMBUS_LOG(ERR, "failed to map channel %d to addr %p", - chan->relid, mapaddr); - vmbus_unmap_resource(mapaddr, file_size); - return -EIO; - } - } - - *ring_size = file_size / 2; - *ring_buf = mapaddr; - - return 0; -} - -int vmbus_uio_map_rings(struct vmbus_channel *chan) -{ - const struct rte_vmbus_device *dev = chan->device; - uint32_t ring_size; - void *ring_buf; - int ret; - - /* Primary channel */ - if (chan->subchannel_id == 0) - ret = vmbus_uio_map_primary(chan, &ring_buf, &ring_size); - else - ret = vmbus_uio_map_subchan(dev, chan, &ring_buf, &ring_size); - - if (ret) - return ret; - - vmbus_br_setup(&chan->txbr, ring_buf, ring_size); - vmbus_br_setup(&chan->rxbr, (char *)ring_buf + ring_size, ring_size); return 0; } @@ -377,18 +117,6 @@ bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, return vmbus_uio_ring_present(dev, chan->relid); } -static bool vmbus_isnew_subchannel(struct vmbus_channel *primary, - unsigned long id) -{ - const struct vmbus_channel *c; - - STAILQ_FOREACH(c, &primary->subchannel_list, next) { - if (c->relid == id) - return false; - } - return true; -} - int vmbus_uio_get_subchan(struct vmbus_channel *primary, struct vmbus_channel **subchan) { diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build index 01ef01f..60913d0 100644 --- a/drivers/bus/vmbus/meson.build +++ b/drivers/bus/vmbus/meson.build @@ -18,7 +18,8 @@ sources = files( includes += include_directories('unix') sources += files( - 'unix/vmbus_unix_bus.c' + 'unix/vmbus_unix_bus.c', + 'unix/vmbus_unix_uio.c' ) if is_linux diff --git a/drivers/bus/vmbus/unix/vmbus_unix.h b/drivers/bus/vmbus/unix/vmbus_unix.h index 2db9399..579c4bb 100644 --- a/drivers/bus/vmbus/unix/vmbus_unix.h +++ b/drivers/bus/vmbus/unix/vmbus_unix.h @@ -6,6 +6,18 @@ #ifndef _VMBUS_BUS_UNIX_H_ #define _VMBUS_BUS_UNIX_H_ +#include <rte_bus_vmbus.h> + +#include "private.h" + extern const rte_uuid_t vmbus_nic_uuid; +extern void *vmbus_map_addr; + +int vmbus_uio_map_subchan_os(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan, + void **mapaddr, size_t *file_size); + +bool vmbus_isnew_subchannel(struct vmbus_channel *primary, + unsigned long id); #endif /* _VMBUS_BUS_UNIX_H_ */ diff --git a/drivers/bus/vmbus/unix/vmbus_unix_uio.c b/drivers/bus/vmbus/unix/vmbus_unix_uio.c new file mode 100644 index 0000000..c5ce8ca --- /dev/null +++ b/drivers/bus/vmbus/unix/vmbus_unix_uio.c @@ -0,0 +1,306 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <unistd.h> +#include <fcntl.h> +#include <sys/mman.h> + +#include <rte_eal.h> +#include <rte_log.h> +#include <rte_bus.h> +#include <rte_memory.h> +#include <rte_common.h> +#include <rte_malloc.h> +#include <rte_bus_vmbus.h> +#include <rte_string_fns.h> + +#include "private.h" +#include "vmbus_unix.h" + +void *vmbus_map_addr; + +/* Control interrupts */ +void vmbus_uio_irq_control(struct rte_vmbus_device *dev, int32_t onoff) +{ + if ((rte_intr_fd_get(dev->intr_handle) < 0) || + write(rte_intr_fd_get(dev->intr_handle), &onoff, + sizeof(onoff)) < 0) { + VMBUS_LOG(ERR, "cannot write to %d:%s", + rte_intr_fd_get(dev->intr_handle), + strerror(errno)); + } +} + +int vmbus_uio_irq_read(struct rte_vmbus_device *dev) +{ + int32_t count; + int cc; + + if (rte_intr_fd_get(dev->intr_handle) < 0) + return -1; + + cc = read(rte_intr_fd_get(dev->intr_handle), &count, + sizeof(count)); + if (cc < (int)sizeof(count)) { + if (cc < 0) { + VMBUS_LOG(ERR, "IRQ read failed %s", + strerror(errno)); + return -errno; + } + VMBUS_LOG(ERR, "can't read IRQ count"); + return -EINVAL; + } + + return count; +} + +void +vmbus_uio_free_resource(struct rte_vmbus_device *dev, + struct mapped_vmbus_resource *uio_res) +{ + rte_free(uio_res); + + if (rte_intr_dev_fd_get(dev->intr_handle) >= 0) { + close(rte_intr_dev_fd_get(dev->intr_handle)); + rte_intr_dev_fd_set(dev->intr_handle, -1); + } + + if (rte_intr_fd_get(dev->intr_handle) >= 0) { + close(rte_intr_fd_get(dev->intr_handle)); + rte_intr_fd_set(dev->intr_handle, -1); + rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UNKNOWN); + } +} + +int +vmbus_uio_alloc_resource(struct rte_vmbus_device *dev, + struct mapped_vmbus_resource **uio_res) +{ + char devname[PATH_MAX]; /* contains the /dev/uioX */ + int fd; + + /* save fd if in primary process */ + snprintf(devname, sizeof(devname), "/dev/uio%u", dev->uio_num); + fd = open(devname, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + devname, strerror(errno)); + goto error; + } + + if (rte_intr_fd_set(dev->intr_handle, fd)) + goto error; + + if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UIO_INTX)) + goto error; + + /* allocate the mapping details for secondary processes*/ + *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); + if (*uio_res == NULL) { + VMBUS_LOG(ERR, "cannot store uio mmap details"); + goto error; + } + + strlcpy((*uio_res)->path, devname, PATH_MAX); + rte_uuid_copy((*uio_res)->id, dev->device_id); + + return 0; + +error: + vmbus_uio_free_resource(dev, *uio_res); + return -1; +} + +static int +find_max_end_va(const struct rte_memseg_list *msl, void *arg) +{ + size_t sz = msl->memseg_arr.len * msl->page_sz; + void *end_va = RTE_PTR_ADD(msl->base_va, sz); + void **max_va = arg; + + if (*max_va < end_va) + *max_va = end_va; + return 0; +} + +/* + * TODO: this should be part of memseg api. + * code is duplicated from PCI. + */ +static void * +vmbus_find_max_end_va(void) +{ + void *va = NULL; + + rte_memseg_list_walk(find_max_end_va, &va); + return va; +} + +int +vmbus_uio_map_resource_by_index(struct rte_vmbus_device *dev, int idx, + struct mapped_vmbus_resource *uio_res, + int flags) +{ + size_t size = dev->resource[idx].len; + struct vmbus_map *maps = uio_res->maps; + void *mapaddr; + off_t offset; + int fd; + + /* devname for mmap */ + fd = open(uio_res->path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + uio_res->path, strerror(errno)); + return -1; + } + + /* try mapping somewhere close to the end of hugepages */ + if (vmbus_map_addr == NULL) + vmbus_map_addr = vmbus_find_max_end_va(); + + /* offset is special in uio it indicates which resource */ + offset = idx * rte_mem_page_size(); + + mapaddr = vmbus_map_resource(vmbus_map_addr, fd, offset, size, flags); + close(fd); + + if (mapaddr == MAP_FAILED) + return -1; + + dev->resource[idx].addr = mapaddr; + vmbus_map_addr = RTE_PTR_ADD(mapaddr, size); + + /* Record result of successful mapping for use by secondary */ + maps[idx].addr = mapaddr; + maps[idx].size = size; + + return 0; +} + +static int vmbus_uio_map_primary(struct vmbus_channel *chan, + void **ring_buf, uint32_t *ring_size) +{ + struct mapped_vmbus_resource *uio_res; + + uio_res = vmbus_uio_find_resource(chan->device); + if (!uio_res) { + VMBUS_LOG(ERR, "can not find resources!"); + return -ENOMEM; + } + + if (uio_res->nb_maps < VMBUS_MAX_RESOURCE) { + VMBUS_LOG(ERR, "VMBUS: only %u resources found!", + uio_res->nb_maps); + return -EINVAL; + } + + *ring_size = uio_res->maps[HV_TXRX_RING_MAP].size / 2; + *ring_buf = uio_res->maps[HV_TXRX_RING_MAP].addr; + return 0; +} + +static int vmbus_uio_map_subchan(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan, + void **ring_buf, uint32_t *ring_size) +{ + size_t file_size = 0; + void *mapaddr = NULL; + struct mapped_vmbus_resource *uio_res; + int channel_idx; + int ret; + + uio_res = vmbus_uio_find_resource(dev); + if (!uio_res) { + VMBUS_LOG(ERR, "can not find resources for mapping subchan"); + return -ENOMEM; + } + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + if (uio_res->nb_subchannels >= UIO_MAX_SUBCHANNEL) { + VMBUS_LOG(ERR, + "exceeding max subchannels UIO_MAX_SUBCHANNEL(%d)", + UIO_MAX_SUBCHANNEL); + VMBUS_LOG(ERR, "Change UIO_MAX_SUBCHANNEL and recompile"); + return -ENOMEM; + } + } else { + for (channel_idx = 0; channel_idx < uio_res->nb_subchannels; + channel_idx++) + if (uio_res->subchannel_maps[channel_idx].relid == + chan->relid) + break; + if (channel_idx == uio_res->nb_subchannels) { + VMBUS_LOG(ERR, + "couldn't find sub channel %d from shared mapping in primary", + chan->relid); + return -ENOMEM; + } + vmbus_map_addr = uio_res->subchannel_maps[channel_idx].addr; + } + + ret = vmbus_uio_map_subchan_os(dev, chan, &mapaddr, &file_size); + if (ret) + return ret; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + + /* Add this mapping to uio_res for use by secondary */ + uio_res->subchannel_maps[uio_res->nb_subchannels].relid = + chan->relid; + uio_res->subchannel_maps[uio_res->nb_subchannels].addr = + mapaddr; + uio_res->subchannel_maps[uio_res->nb_subchannels].size = + file_size; + uio_res->nb_subchannels++; + + vmbus_map_addr = RTE_PTR_ADD(mapaddr, file_size); + } else { + if (mapaddr != vmbus_map_addr) { + VMBUS_LOG(ERR, "failed to map channel %d to addr %p", + chan->relid, mapaddr); + vmbus_unmap_resource(mapaddr, file_size); + return -EIO; + } + } + + *ring_size = file_size / 2; + *ring_buf = mapaddr; + + return 0; +} + +int vmbus_uio_map_rings(struct vmbus_channel *chan) +{ + const struct rte_vmbus_device *dev = chan->device; + uint32_t ring_size = 0; + void *ring_buf = NULL; + int ret; + + /* Primary channel */ + if (chan->subchannel_id == 0) + ret = vmbus_uio_map_primary(chan, &ring_buf, &ring_size); + else + ret = vmbus_uio_map_subchan(dev, chan, &ring_buf, &ring_size); + + if (ret) + return ret; + + vmbus_br_setup(&chan->txbr, ring_buf, ring_size); + vmbus_br_setup(&chan->rxbr, (char *)ring_buf + ring_size, ring_size); + return 0; +} + +bool vmbus_isnew_subchannel(struct vmbus_channel *primary, + unsigned long id) +{ + const struct vmbus_channel *c; + + STAILQ_FOREACH(c, &primary->subchannel_list, next) { + if (c->relid == id) + return false; + } + return true; +} -- 1.8.3.1
Using sysctl, all the devices on the VMBUS are identified by the PMD. On finding the Network device's device id, it is added to VMBUS dev list. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_bus.c | 268 ++++++++++++++++++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 drivers/bus/vmbus/freebsd/vmbus_bus.c diff --git a/drivers/bus/vmbus/freebsd/vmbus_bus.c b/drivers/bus/vmbus/freebsd/vmbus_bus.c new file mode 100644 index 0000000..c1a3a5f --- /dev/null +++ b/drivers/bus/vmbus/freebsd/vmbus_bus.c @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <rte_eal.h> +#include <rte_bus_vmbus.h> + +#include "private.h" +#include "vmbus_unix.h" + +#include <sys/bus.h> +#include <sys/types.h> +#include <sys/sysctl.h> + +/* Parse UUID. Caller must pass NULL terminated string */ +static int +parse_sysfs_uuid(const char *filename, rte_uuid_t uu) +{ + char in[BUFSIZ]; + + memcpy(in, filename, BUFSIZ); + if (rte_uuid_parse(in, uu) < 0) { + VMBUS_LOG(ERR, "%s not a valid UUID", in); + return -1; + } + + return 0; +} + +/* Scan one vmbus entry, and fill the devices list from it. */ +static int +vmbus_scan_one(const char *name, unsigned int unit_num) +{ + struct rte_vmbus_device *dev, *dev2; + char sysctl_buf[PATH_MAX], sysctl_var[PATH_MAX]; + size_t guid_len = 36, len = PATH_MAX; + char classid[guid_len + 1], deviceid[guid_len + 1]; + + dev = calloc(1, sizeof(*dev)); + if (dev == NULL) + return -1; + + /* get class id and device id */ + snprintf(sysctl_var, len, "dev.%s.%u.%%pnpinfo", name, unit_num); + if (sysctlbyname(sysctl_var, &sysctl_buf, &len, NULL, 0) < 0) + goto error; + + /* pnpinfo: classid=f912ad6d-2b17-48ea-bd65-f927a61c7684 + * deviceid=d34b2567-b9b6-42b9-8778-0a4ec0b955bf + */ + if (sysctl_buf[0] == 'c' && sysctl_buf[1] == 'l' && + sysctl_buf[7] == '=') { + memcpy(classid, &sysctl_buf[8], guid_len); + classid[guid_len] = '\0'; + } + if (parse_sysfs_uuid(classid, dev->class_id) < 0) + goto error; + + /* skip non-network devices */ + if (rte_uuid_compare(dev->class_id, vmbus_nic_uuid) != 0) { + free(dev); + return 0; + } + + if (sysctl_buf[45] == 'd' && sysctl_buf[46] == 'e' && + sysctl_buf[47] == 'v' && sysctl_buf[53] == '=') { + memcpy(deviceid, &sysctl_buf[54], guid_len); + deviceid[guid_len] = '\0'; + } + if (parse_sysfs_uuid(deviceid, dev->device_id) < 0) + goto error; + + if (!strcmp(name, "hv_uio")) + dev->uio_num = unit_num; + else + dev->uio_num = -1; + dev->device.bus = &rte_vmbus_bus.bus; + dev->device.numa_node = 0; + dev->device.name = strdup(deviceid); + if (!dev->device.name) + goto error; + + dev->device.devargs = vmbus_devargs_lookup(dev); + + dev->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE); + if (dev->intr_handle == NULL) + goto error; + + /* device is valid, add in list (sorted) */ + VMBUS_LOG(DEBUG, "Adding vmbus device %s", name); + + TAILQ_FOREACH(dev2, &rte_vmbus_bus.device_list, next) { + int ret; + + ret = rte_uuid_compare(dev->device_id, dev2->device_id); + if (ret > 0) + continue; + + if (ret < 0) { + vmbus_insert_device(dev2, dev); + } else { /* already registered */ + VMBUS_LOG(NOTICE, + "%s already registered", name); + free(dev); + } + return 0; + } + + vmbus_add_device(dev); + return 0; +error: + VMBUS_LOG(DEBUG, "failed"); + + free(dev); + return -1; +} + +static int +vmbus_unpack(char *walker, char *ep, char **str) +{ + int ret = 0; + + *str = strdup(walker); + if (*str == NULL) { + ret = -ENOMEM; + goto exit; + } + + if (walker + strnlen(walker, ep - walker) >= ep) { + ret = -EINVAL; + goto exit; + } +exit: + return ret; +} + +/* + * Scan the content of the vmbus, and the devices in the devices list + */ +int +rte_vmbus_scan(void) +{ + struct u_device udev; + struct u_businfo ubus; + int dev_idx, dev_ptr, name2oid[2], oid[CTL_MAXNAME + 12], error; + size_t oidlen, rlen, ub_size; + uintptr_t vmbus_handle = 0; + char *walker, *ep; + char name[16] = "hw.bus.devices"; + char *dd_name, *dd_desc, *dd_drivername, *dd_pnpinfo, *dd_location; + + /* + * devinfo FreeBSD APP logic to fetch all the VMBus devices + * using SYSCTLs + */ + name2oid[0] = 0; + name2oid[1] = 3; + oidlen = sizeof(oid); + error = sysctl(name2oid, 2, oid, &oidlen, name, strlen(name)); + if (error < 0) { + VMBUS_LOG(DEBUG, "can't find hw.bus.devices sysctl node"); + return -ENOENT; + } + oidlen /= sizeof(int); + if (oidlen > CTL_MAXNAME) { + VMBUS_LOG(DEBUG, "hw.bus.devices oid is too large"); + return -EINVAL; + } + + ub_size = sizeof(ubus); + if (sysctlbyname("hw.bus.info", &ubus, &ub_size, NULL, 0) != 0) { + VMBUS_LOG(DEBUG, "sysctlbyname(\"hw.bus.info\", ...) failed"); + return -EINVAL; + } + if ((ub_size != sizeof(ubus)) || + (ubus.ub_version != BUS_USER_VERSION)) { + VMBUS_LOG(DEBUG, + "kernel bus interface version mismatch: kernel %d expected %d", + ubus.ub_version, BUS_USER_VERSION); + return -EINVAL; + } + + oid[oidlen++] = ubus.ub_generation; + dev_ptr = oidlen++; + + /* + * Scan devices. + * + * Stop after a fairly insane number to avoid death in the case + * of kernel corruption. + */ + + for (dev_idx = 0; dev_idx < 10000; dev_idx++) { + /* + * Get the device information. + */ + oid[dev_ptr] = dev_idx; + rlen = sizeof(udev); + error = sysctl(oid, oidlen, &udev, &rlen, NULL, 0); + if (error < 0) { + if (errno == ENOENT) /* end of list */ + break; + if (errno != EINVAL) /* gen count skip, restart */ + VMBUS_LOG(DEBUG, "sysctl hw.bus.devices.%d", + dev_idx); + return errno; + } + if (rlen != sizeof(udev)) { + VMBUS_LOG(DEBUG, + "sysctl returned wrong data %zd bytes instead of %zd", + rlen, sizeof(udev)); + return -EINVAL; + } + + walker = udev.dv_fields; + ep = walker + sizeof(udev.dv_fields); + dd_name = NULL; + dd_desc = NULL; + dd_drivername = NULL; + dd_pnpinfo = NULL; + dd_location = NULL; + + error = vmbus_unpack(walker, ep, &dd_name); + if (error < 0) + return error; + walker += strlen(walker) + 1; + + error = vmbus_unpack(walker, ep, &dd_desc); + if (error < 0) + return error; + walker += strlen(walker) + 1; + + error = vmbus_unpack(walker, ep, &dd_drivername); + if (error < 0) + return error; + walker += strlen(walker) + 1; + + error = vmbus_unpack(walker, ep, &dd_pnpinfo); + if (error < 0) + return error; + walker += strlen(walker) + 1; + + error = vmbus_unpack(walker, ep, &dd_location); + if (error < 0) + return error; + + if (*dd_drivername && !(strcmp(dd_drivername, "vmbus"))) + vmbus_handle = udev.dv_handle; + + if (vmbus_handle && (vmbus_handle == udev.dv_parent) + && *dd_pnpinfo && *dd_name) { + unsigned int driver_len = 0, unit_num = 0; + char *endptr; + + driver_len = strlen(dd_drivername); + unit_num = strtoull(&dd_name[driver_len], &endptr, 10); + VMBUS_LOG(DEBUG, "Device name:%s, pnpinfo:%s", + dd_name, dd_pnpinfo); + + if (vmbus_scan_one(dd_drivername, unit_num) < 0) + goto error; + } + } + return 0; +error: + return -1; +} -- 1.8.3.1
All resource values are published by HV_UIO driver as sysctl key value pairs and they are read at a later point of the code flow Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_bus.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_bus.c b/drivers/bus/vmbus/freebsd/vmbus_bus.c index c1a3a5f..28f5ff4 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_bus.c +++ b/drivers/bus/vmbus/freebsd/vmbus_bus.c @@ -28,6 +28,24 @@ return 0; } +/* map the resources of a vmbus device in virtual memory */ +int +rte_vmbus_map_device(struct rte_vmbus_device *dev) +{ + if (dev->uio_num < 0) { + VMBUS_LOG(DEBUG, "Not managed by UIO driver, skipped"); + return 1; + } + + return vmbus_uio_map_resource(dev); +} + +void +rte_vmbus_unmap_device(struct rte_vmbus_device *dev) +{ + vmbus_uio_unmap_resource(dev); +} + /* Scan one vmbus entry, and fill the devices list from it. */ static int vmbus_scan_one(const char *name, unsigned int unit_num) -- 1.8.3.1
The UIO device's attribute (relid, monitor id, etc) values are retrieved using following sysctl variables: $ sysctl dev.hv_uio.0 dev.hv_uio.0.send_buf.gpadl: 925241 dev.hv_uio.0.send_buf.size: 16777216 dev.hv_uio.0.recv_buf.gpadl: 925240 dev.hv_uio.0.recv_buf.size: 32505856 dev.hv_uio.0.monitor_page.size: 4096 dev.hv_uio.0.int_page.size: 4096 Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 105 ++++++++++++++++++++++++++++++++ drivers/bus/vmbus/linux/vmbus_uio.c | 16 +++++ drivers/bus/vmbus/unix/vmbus_unix.h | 4 ++ drivers/bus/vmbus/unix/vmbus_unix_uio.c | 6 +- 4 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 drivers/bus/vmbus/freebsd/vmbus_uio.c diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c new file mode 100644 index 0000000..0544371 --- /dev/null +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018, Microsoft Corporation. + * All Rights Reserved. + */ + +#include <unistd.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/types.h> +#include <sys/sysctl.h> +#include <sys/ioctl.h> + +#include <rte_log.h> +#include <rte_bus.h> +#include <rte_malloc.h> +#include <rte_bus_vmbus.h> + +#include "private.h" +#include "vmbus_unix.h" + +const char *driver_name = "hv_uio"; + +/* Check map names with kernel names */ +static const char *map_names[VMBUS_MAX_RESOURCE] = { + [HV_TXRX_RING_MAP] = "txrx_rings", + [HV_INT_PAGE_MAP] = "int_page", + [HV_MON_PAGE_MAP] = "monitor_page", + [HV_RECV_BUF_MAP] = "recv_buf", + [HV_SEND_BUF_MAP] = "send_buf", +}; + +static int +sysctl_get_vmbus_device_info(struct rte_vmbus_device *dev) +{ + char sysctl_buf[PATH_MAX]; + char sysctl_var[PATH_MAX]; + size_t len = PATH_MAX, sysctl_len; + unsigned long tmp; + int i; + + snprintf(sysctl_buf, len, "dev.%s.%d", driver_name, dev->uio_num); + + sysctl_len = sizeof(unsigned long); + /* get relid */ + snprintf(sysctl_var, len, "%s.channel.ch_id", sysctl_buf); + if (sysctlbyname(sysctl_var, &tmp, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s", sysctl_var); + goto error; + } + dev->relid = tmp; + + /* get monitor id */ + snprintf(sysctl_var, len, "%s.channel.%u.monitor_id", sysctl_buf, + dev->relid); + if (sysctlbyname(sysctl_var, &tmp, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s", sysctl_var); + goto error; + } + dev->monitor_id = tmp; + + /* Extract resource value */ + for (i = 0; i < VMBUS_MAX_RESOURCE; i++) { + struct rte_mem_resource *res = &dev->resource[i]; + unsigned long size, gpad = 0; + size_t sizelen = sizeof(len); + + snprintf(sysctl_var, sizeof(sysctl_var), "%s.%s.size", + sysctl_buf, map_names[i]); + if (sysctlbyname(sysctl_var, &size, &sizelen, NULL, 0) < 0) { + VMBUS_LOG(ERR, + "could not read %s", sysctl_var); + goto error; + } + res->len = size; + + if (i == HV_RECV_BUF_MAP || i == HV_SEND_BUF_MAP) { + snprintf(sysctl_var, sizeof(sysctl_var), "%s.%s.gpadl", + sysctl_buf, map_names[i]); + if (sysctlbyname(sysctl_var, &gpad, &sizelen, NULL, 0) < 0) { + VMBUS_LOG(ERR, + "could not read %s", sysctl_var); + goto error; + } + /* put the GPAD value in physical address */ + res->phys_addr = gpad; + } + } + return 0; +error: + return -1; +} + +/* + * On FreeBSD, the device is opened first to ensure kernel UIO driver + * is properly initialized before reading device attributes + */ +int vmbus_get_device_info_os(struct rte_vmbus_device *dev) +{ + return sysctl_get_vmbus_device_info(dev); +} + +const char *get_devname_os(void) +{ + return "/dev/hv_uio"; +} diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c b/drivers/bus/vmbus/linux/vmbus_uio.c index b5d15c9..69f0b26 100644 --- a/drivers/bus/vmbus/linux/vmbus_uio.c +++ b/drivers/bus/vmbus/linux/vmbus_uio.c @@ -199,3 +199,19 @@ int vmbus_uio_get_subchan(struct vmbus_channel *primary, closedir(chan_dir); return err; } + +/* + * In Linux the device info is fetched from SYSFS and doesn't need + * opening of the device before reading its attributes + * This is a stub function and it should always succeed. + */ +int vmbus_get_device_info_os(struct rte_vmbus_device *dev) +{ + RTE_SET_USED(dev); + return 0; +} + +const char *get_devname_os(void) +{ + return "/dev/uio"; +} diff --git a/drivers/bus/vmbus/unix/vmbus_unix.h b/drivers/bus/vmbus/unix/vmbus_unix.h index 579c4bb..59afc10 100644 --- a/drivers/bus/vmbus/unix/vmbus_unix.h +++ b/drivers/bus/vmbus/unix/vmbus_unix.h @@ -20,4 +20,8 @@ int vmbus_uio_map_subchan_os(const struct rte_vmbus_device *dev, bool vmbus_isnew_subchannel(struct vmbus_channel *primary, unsigned long id); +int vmbus_get_device_info_os(struct rte_vmbus_device *dev); + +const char *get_devname_os(void); + #endif /* _VMBUS_BUS_UNIX_H_ */ diff --git a/drivers/bus/vmbus/unix/vmbus_unix_uio.c b/drivers/bus/vmbus/unix/vmbus_unix_uio.c index c5ce8ca..5a57a5e 100644 --- a/drivers/bus/vmbus/unix/vmbus_unix_uio.c +++ b/drivers/bus/vmbus/unix/vmbus_unix_uio.c @@ -82,7 +82,8 @@ int vmbus_uio_irq_read(struct rte_vmbus_device *dev) int fd; /* save fd if in primary process */ - snprintf(devname, sizeof(devname), "/dev/uio%u", dev->uio_num); + snprintf(devname, sizeof(devname), "%s%u", get_devname_os(), + dev->uio_num); fd = open(devname, O_RDWR); if (fd < 0) { VMBUS_LOG(ERR, "Cannot open %s: %s", @@ -106,6 +107,9 @@ int vmbus_uio_irq_read(struct rte_vmbus_device *dev) strlcpy((*uio_res)->path, devname, PATH_MAX); rte_uuid_copy((*uio_res)->id, dev->device_id); + if (vmbus_get_device_info_os(dev) < 0) + goto error; + return 0; error: -- 1.8.3.1
- Event monitoring is not yet supported on FreeBSD, hence moving it to the OS specific files - Add meson support to OS environment Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/freebsd/hn_os.c | 16 ++++++++++++++++ drivers/net/netvsc/freebsd/meson.build | 6 ++++++ drivers/net/netvsc/hn_ethdev.c | 7 +++---- drivers/net/netvsc/hn_os.h | 6 ++++++ drivers/net/netvsc/linux/hn_os.c | 21 +++++++++++++++++++++ drivers/net/netvsc/linux/meson.build | 6 ++++++ drivers/net/netvsc/meson.build | 3 +++ 7 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 drivers/net/netvsc/freebsd/hn_os.c create mode 100644 drivers/net/netvsc/freebsd/meson.build create mode 100644 drivers/net/netvsc/hn_os.h create mode 100644 drivers/net/netvsc/linux/hn_os.c create mode 100644 drivers/net/netvsc/linux/meson.build diff --git a/drivers/net/netvsc/freebsd/hn_os.c b/drivers/net/netvsc/freebsd/hn_os.c new file mode 100644 index 0000000..4c6a798 --- /dev/null +++ b/drivers/net/netvsc/freebsd/hn_os.c @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2021 Microsoft Corporation + */ + +#include <stdio.h> + +#include <rte_common.h> + +#include "hn_logs.h" +#include "hn_os.h" + +int eth_hn_os_dev_event(void) +{ + PMD_DRV_LOG(DEBUG, "rte_dev_event_monitor_start not supported on FreeBSD"); + return 0; +} diff --git a/drivers/net/netvsc/freebsd/meson.build b/drivers/net/netvsc/freebsd/meson.build new file mode 100644 index 0000000..78f824f --- /dev/null +++ b/drivers/net/netvsc/freebsd/meson.build @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Microsoft Corporation + +sources += files( + 'hn_os.c', +) diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c index 8a95040..8b1e07b 100644 --- a/drivers/net/netvsc/hn_ethdev.c +++ b/drivers/net/netvsc/hn_ethdev.c @@ -39,6 +39,7 @@ #include "hn_rndis.h" #include "hn_nvs.h" #include "ndis.h" +#include "hn_os.h" #define HN_TX_OFFLOAD_CAPS (RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | \ RTE_ETH_TX_OFFLOAD_TCP_CKSUM | \ @@ -1240,11 +1241,9 @@ static int eth_hn_probe(struct rte_vmbus_driver *drv __rte_unused, PMD_INIT_FUNC_TRACE(); - ret = rte_dev_event_monitor_start(); - if (ret) { - PMD_DRV_LOG(ERR, "Failed to start device event monitoring"); + ret = eth_hn_os_dev_event(); + if (ret) return ret; - } eth_dev = eth_dev_vmbus_allocate(dev, sizeof(struct hn_data)); if (!eth_dev) diff --git a/drivers/net/netvsc/hn_os.h b/drivers/net/netvsc/hn_os.h new file mode 100644 index 0000000..618c53c --- /dev/null +++ b/drivers/net/netvsc/hn_os.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2009-2021 Microsoft Corp. + * All rights reserved. + */ + +int eth_hn_os_dev_event(void); diff --git a/drivers/net/netvsc/linux/hn_os.c b/drivers/net/netvsc/linux/hn_os.c new file mode 100644 index 0000000..1ea12ce --- /dev/null +++ b/drivers/net/netvsc/linux/hn_os.c @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2021 Microsoft Corporation + */ + +#include <unistd.h> + +#include <rte_ethdev.h> + +#include "hn_logs.h" +#include "hn_os.h" + +int eth_hn_os_dev_event(void) +{ + int ret; + + ret = rte_dev_event_monitor_start(); + if (ret) + PMD_DRV_LOG(ERR, "Failed to start device event monitoring"); + + return ret; +} diff --git a/drivers/net/netvsc/linux/meson.build b/drivers/net/netvsc/linux/meson.build new file mode 100644 index 0000000..78f824f --- /dev/null +++ b/drivers/net/netvsc/linux/meson.build @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Microsoft Corporation + +sources += files( + 'hn_os.c', +) diff --git a/drivers/net/netvsc/meson.build b/drivers/net/netvsc/meson.build index bb6225d..c50414d 100644 --- a/drivers/net/netvsc/meson.build +++ b/drivers/net/netvsc/meson.build @@ -8,6 +8,7 @@ if is_windows endif deps += 'bus_vmbus' +includes += include_directories(exec_env) sources = files( 'hn_ethdev.c', 'hn_nvs.c', @@ -15,3 +16,5 @@ sources = files( 'hn_rxtx.c', 'hn_vf.c', ) + +subdir(exec_env) -- 1.8.3.1
To map the subchannels, an mmap request is directly made after determining the subchan memory offset Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 48 +++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index 0544371..55b8f18 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -18,6 +18,13 @@ #include "private.h" #include "vmbus_unix.h" +/* + * Macros to distinguish mmap request + * [7-0] - Device memory region + * [15-8]- Sub-channel id + */ +#define UH_SUBCHAN_MASK_SHIFT 8 + const char *driver_name = "hv_uio"; /* Check map names with kernel names */ @@ -99,6 +106,47 @@ int vmbus_get_device_info_os(struct rte_vmbus_device *dev) return sysctl_get_vmbus_device_info(dev); } +int vmbus_uio_map_subchan_os(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan, + void **mapaddr, size_t *size) +{ + char ring_path[PATH_MAX]; + off_t offset; + int fd; + + snprintf(ring_path, sizeof(ring_path), + "/dev/hv_uio%d", dev->uio_num); + + fd = open(ring_path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + ring_path, strerror(errno)); + return -errno; + } + + /* subchannel rings are of the same size as primary */ + *size = dev->resource[HV_TXRX_RING_MAP].len; + offset = (chan->relid << UH_SUBCHAN_MASK_SHIFT) * PAGE_SIZE; + + *mapaddr = vmbus_map_resource(vmbus_map_addr, fd, + offset, *size, 0); + close(fd); + + if (*mapaddr == MAP_FAILED) + return -EIO; + + return 0; +} + +/* This function should always succeed */ +bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, + const struct vmbus_channel *chan) +{ + RTE_SET_USED(dev); + RTE_SET_USED(chan); + return true; +} + const char *get_devname_os(void) { return "/dev/hv_uio"; -- 1.8.3.1
In FreeBSD, unlike Linux there is no sub-channel open callback that could be called by HV_UIO driver upon their grant by the hypervisor. Thus the PMD makes an IOCTL to the HV_UIO to open the sub-channels On Linux, the vmbus_uio_subchan_open() will always return success as the Linux HV_UIO opens them implicitly. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 30 ++++++++++++++++++++++++++++++ drivers/bus/vmbus/linux/vmbus_uio.c | 12 ++++++++++++ drivers/bus/vmbus/private.h | 1 + drivers/bus/vmbus/rte_bus_vmbus.h | 11 +++++++++++ drivers/bus/vmbus/version.map | 6 ++++++ drivers/bus/vmbus/vmbus_channel.c | 5 +++++ 6 files changed, 65 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index 55b8f18..438db41 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -25,6 +25,9 @@ */ #define UH_SUBCHAN_MASK_SHIFT 8 +/* ioctl */ +#define HVIOOPENSUBCHAN _IOW('h', 14, uint32_t) + const char *driver_name = "hv_uio"; /* Check map names with kernel names */ @@ -151,3 +154,30 @@ const char *get_devname_os(void) { return "/dev/hv_uio"; } + +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, uint32_t subchan) +{ + struct mapped_vmbus_resource *uio_res; + int fd, err = 0; + + uio_res = vmbus_uio_find_resource(dev); + if (!uio_res) { + VMBUS_LOG(ERR, "cannot find uio resource"); + return -EINVAL; + } + + fd = open(uio_res->path, O_RDWR); + if (fd < 0) { + VMBUS_LOG(ERR, "Cannot open %s: %s", + uio_res->path, strerror(errno)); + return -1; + } + + if (ioctl(fd, HVIOOPENSUBCHAN, &subchan)) { + VMBUS_LOG(ERR, "open subchan ioctl failed %s: %s", + uio_res->path, strerror(errno)); + err = -1; + } + close(fd); + return err; +} diff --git a/drivers/bus/vmbus/linux/vmbus_uio.c b/drivers/bus/vmbus/linux/vmbus_uio.c index 69f0b26..b9616bd 100644 --- a/drivers/bus/vmbus/linux/vmbus_uio.c +++ b/drivers/bus/vmbus/linux/vmbus_uio.c @@ -215,3 +215,15 @@ const char *get_devname_os(void) { return "/dev/uio"; } + +/* + * This is a stub function and it should always succeed. + * The Linux UIO kernel driver opens the subchannels implicitly. + */ +int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, + uint32_t subchan) +{ + RTE_SET_USED(dev); + RTE_SET_USED(subchan); + return 0; +} diff --git a/drivers/bus/vmbus/private.h b/drivers/bus/vmbus/private.h index 1bca147..ea0276a 100644 --- a/drivers/bus/vmbus/private.h +++ b/drivers/bus/vmbus/private.h @@ -116,6 +116,7 @@ bool vmbus_uio_subchannels_supported(const struct rte_vmbus_device *dev, int vmbus_uio_get_subchan(struct vmbus_channel *primary, struct vmbus_channel **subchan); int vmbus_uio_map_rings(struct vmbus_channel *chan); +int vmbus_uio_subchan_open(struct rte_vmbus_device *device, uint32_t subchan); void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen); diff --git a/drivers/bus/vmbus/rte_bus_vmbus.h b/drivers/bus/vmbus/rte_bus_vmbus.h index a24bad8..06b2ffc 100644 --- a/drivers/bus/vmbus/rte_bus_vmbus.h +++ b/drivers/bus/vmbus/rte_bus_vmbus.h @@ -404,6 +404,17 @@ void rte_vmbus_set_latency(const struct rte_vmbus_device *dev, */ void rte_vmbus_unregister(struct rte_vmbus_driver *driver); +/** + * Perform IOCTL to VMBUS device + * + * @param device + * A pointer to a rte_vmbus_device structure + * @param subchan + * Count of subchannels to open + */ +__rte_experimental +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan); + /** Helper for VMBUS device registration from driver instance */ #define RTE_PMD_REGISTER_VMBUS(nm, vmbus_drv) \ RTE_INIT(vmbusinitfn_ ##nm) \ diff --git a/drivers/bus/vmbus/version.map b/drivers/bus/vmbus/version.map index 3cadec7..e5b7218 100644 --- a/drivers/bus/vmbus/version.map +++ b/drivers/bus/vmbus/version.map @@ -26,3 +26,9 @@ DPDK_22 { local: *; }; + +EXPERIMENTAL { + global: + + rte_vmbus_ioctl; +}; diff --git a/drivers/bus/vmbus/vmbus_channel.c b/drivers/bus/vmbus/vmbus_channel.c index 119b9b3..9a8f6e3 100644 --- a/drivers/bus/vmbus/vmbus_channel.c +++ b/drivers/bus/vmbus/vmbus_channel.c @@ -365,6 +365,11 @@ int rte_vmbus_max_channels(const struct rte_vmbus_device *device) return 1; } +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan) +{ + return vmbus_uio_subchan_open(device, subchan); +} + /* Setup secondary channel */ int rte_vmbus_subchan_open(struct vmbus_channel *primary, struct vmbus_channel **new_chan) -- 1.8.3.1
make IOCTL call to open subchannels Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/hn_ethdev.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c index 8b1e07b..104c7ae 100644 --- a/drivers/net/netvsc/hn_ethdev.c +++ b/drivers/net/netvsc/hn_ethdev.c @@ -516,6 +516,10 @@ static int hn_subchan_configure(struct hn_data *hv, if (err) return err; + err = rte_vmbus_ioctl(hv->vmbus, subchan); + if (err) + return err; + while (subchan > 0) { struct vmbus_channel *new_sc; uint16_t chn_index; -- 1.8.3.1
Using sysctl, all the subchannel's attributes are fetched Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/freebsd/vmbus_uio.c | 73 +++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/drivers/bus/vmbus/freebsd/vmbus_uio.c b/drivers/bus/vmbus/freebsd/vmbus_uio.c index 438db41..6a9a196 100644 --- a/drivers/bus/vmbus/freebsd/vmbus_uio.c +++ b/drivers/bus/vmbus/freebsd/vmbus_uio.c @@ -155,6 +155,79 @@ const char *get_devname_os(void) return "/dev/hv_uio"; } +int vmbus_uio_get_subchan(struct vmbus_channel *primary, + struct vmbus_channel **subchan) +{ + const struct rte_vmbus_device *dev = primary->device; + char sysctl_buf[PATH_MAX], sysctl_var[PATH_MAX]; + size_t len = PATH_MAX, sysctl_len; + /* nr_schan, relid, subid & monid datatype must match kernel's for sysctl */ + uint32_t relid, subid, nr_schan, i; + uint8_t monid; + int err; + + /* get no. of sub-channels opened by hv_uio + * dev.hv_uio.0.subchan_cnt + */ + snprintf(sysctl_var, len, "dev.%s.%d.subchan_cnt", driver_name, + dev->uio_num); + sysctl_len = sizeof(nr_schan); + if (sysctlbyname(sysctl_var, &nr_schan, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctl_var, + strerror(errno)); + return -1; + } + + /* dev.hv_uio.0.channel.14.sub */ + snprintf(sysctl_buf, len, "dev.%s.%d.channel.%u.sub", driver_name, + dev->uio_num, primary->relid); + for (i = 1; i <= nr_schan; i++) { + /* get relid */ + snprintf(sysctl_var, len, "%s.%u.chanid", sysctl_buf, i); + sysctl_len = sizeof(relid); + if (sysctlbyname(sysctl_var, &relid, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctl_var, + strerror(errno)); + goto error; + } + + if (!vmbus_isnew_subchannel(primary, (uint16_t)relid)) { + VMBUS_LOG(DEBUG, "skip already found channel: %u", + relid); + continue; + } + + /* get sub-channel id */ + snprintf(sysctl_var, len, "%s.%u.ch_subidx", sysctl_buf, i); + sysctl_len = sizeof(subid); + if (sysctlbyname(sysctl_var, &subid, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctl_var, + strerror(errno)); + goto error; + } + + /* get monitor id */ + snprintf(sysctl_var, len, "%s.%u.monitor_id", sysctl_buf, i); + sysctl_len = sizeof(monid); + if (sysctlbyname(sysctl_var, &monid, &sysctl_len, NULL, 0) < 0) { + VMBUS_LOG(ERR, "could not read %s : %s", sysctl_var, + strerror(errno)); + goto error; + } + + err = vmbus_chan_create(dev, (uint16_t)relid, (uint16_t)subid, + monid, subchan); + if (err) { + VMBUS_LOG(ERR, "subchannel setup failed"); + return err; + } + break; + } + return 0; +error: + return -1; +} + int vmbus_uio_subchan_open(struct rte_vmbus_device *dev, uint32_t subchan) { struct mapped_vmbus_resource *uio_res; -- 1.8.3.1
Moved netvsc_hotplug_retry to respective OS dir as it contains OS dependent code. For Linux, it is copied as is and for FreeBSD it is not supported yet. Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/net/netvsc/freebsd/hn_os.c | 5 +++ drivers/net/netvsc/hn_ethdev.c | 84 ----------------------------------- drivers/net/netvsc/hn_os.h | 2 + drivers/net/netvsc/linux/hn_os.c | 90 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 84 deletions(-) diff --git a/drivers/net/netvsc/freebsd/hn_os.c b/drivers/net/netvsc/freebsd/hn_os.c index 4c6a798..fece1be 100644 --- a/drivers/net/netvsc/freebsd/hn_os.c +++ b/drivers/net/netvsc/freebsd/hn_os.c @@ -14,3 +14,8 @@ int eth_hn_os_dev_event(void) PMD_DRV_LOG(DEBUG, "rte_dev_event_monitor_start not supported on FreeBSD"); return 0; } + +void netvsc_hotplug_retry(void *args) +{ + RTE_SET_USED(args); +} diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c index 104c7ae..dd4b872 100644 --- a/drivers/net/netvsc/hn_ethdev.c +++ b/drivers/net/netvsc/hn_ethdev.c @@ -57,9 +57,6 @@ #define NETVSC_ARG_TXBREAK "tx_copybreak" #define NETVSC_ARG_RX_EXTMBUF_ENABLE "rx_extmbuf_enable" -/* The max number of retry when hot adding a VF device */ -#define NETVSC_MAX_HOTADD_RETRY 10 - struct hn_xstats_name_off { char name[RTE_ETH_XSTATS_NAME_SIZE]; unsigned int offset; @@ -556,87 +553,6 @@ static int hn_subchan_configure(struct hn_data *hv, return err; } -static void netvsc_hotplug_retry(void *args) -{ - int ret; - struct hn_data *hv = args; - struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id]; - struct rte_devargs *d = &hv->devargs; - char buf[256]; - - DIR *di; - struct dirent *dir; - struct ifreq req; - struct rte_ether_addr eth_addr; - int s; - - PMD_DRV_LOG(DEBUG, "%s: retry count %d", - __func__, hv->eal_hot_plug_retry); - - if (hv->eal_hot_plug_retry++ > NETVSC_MAX_HOTADD_RETRY) - return; - - snprintf(buf, sizeof(buf), "/sys/bus/pci/devices/%s/net", d->name); - di = opendir(buf); - if (!di) { - PMD_DRV_LOG(DEBUG, "%s: can't open directory %s, " - "retrying in 1 second", __func__, buf); - goto retry; - } - - while ((dir = readdir(di))) { - /* Skip . and .. directories */ - if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) - continue; - - /* trying to get mac address if this is a network device*/ - s = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); - if (s == -1) { - PMD_DRV_LOG(ERR, "Failed to create socket errno %d", - errno); - break; - } - strlcpy(req.ifr_name, dir->d_name, sizeof(req.ifr_name)); - ret = ioctl(s, SIOCGIFHWADDR, &req); - close(s); - if (ret == -1) { - PMD_DRV_LOG(ERR, - "Failed to send SIOCGIFHWADDR for device %s", - dir->d_name); - break; - } - if (req.ifr_hwaddr.sa_family != ARPHRD_ETHER) { - closedir(di); - return; - } - memcpy(eth_addr.addr_bytes, req.ifr_hwaddr.sa_data, - RTE_DIM(eth_addr.addr_bytes)); - - if (rte_is_same_ether_addr(ð_addr, dev->data->mac_addrs)) { - PMD_DRV_LOG(NOTICE, - "Found matching MAC address, adding device %s network name %s", - d->name, dir->d_name); - ret = rte_eal_hotplug_add(d->bus->name, d->name, - d->args); - if (ret) { - PMD_DRV_LOG(ERR, - "Failed to add PCI device %s", - d->name); - break; - } - } - /* When the code reaches here, we either have already added - * the device, or its MAC address did not match. - */ - closedir(di); - return; - } - closedir(di); -retry: - /* The device is still being initialized, retry after 1 second */ - rte_eal_alarm_set(1000000, netvsc_hotplug_retry, hv); -} - static void netvsc_hotadd_callback(const char *device_name, enum rte_dev_event_type type, void *arg) diff --git a/drivers/net/netvsc/hn_os.h b/drivers/net/netvsc/hn_os.h index 618c53c..1fb7292 100644 --- a/drivers/net/netvsc/hn_os.h +++ b/drivers/net/netvsc/hn_os.h @@ -4,3 +4,5 @@ */ int eth_hn_os_dev_event(void); + +void netvsc_hotplug_retry(void *args); diff --git a/drivers/net/netvsc/linux/hn_os.c b/drivers/net/netvsc/linux/hn_os.c index 1ea12ce..9c2f4cd 100644 --- a/drivers/net/netvsc/linux/hn_os.c +++ b/drivers/net/netvsc/linux/hn_os.c @@ -3,12 +3,21 @@ */ #include <unistd.h> +#include <dirent.h> +#include <net/if.h> +#include <net/if_arp.h> +#include <sys/ioctl.h> #include <rte_ethdev.h> +#include <rte_alarm.h> #include "hn_logs.h" +#include "hn_var.h" #include "hn_os.h" +/* The max number of retry when hot adding a VF device */ +#define NETVSC_MAX_HOTADD_RETRY 10 + int eth_hn_os_dev_event(void) { int ret; @@ -19,3 +28,84 @@ int eth_hn_os_dev_event(void) return ret; } + +void netvsc_hotplug_retry(void *args) +{ + int ret; + struct hn_data *hv = args; + struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id]; + struct rte_devargs *d = &hv->devargs; + char buf[256]; + + DIR *di; + struct dirent *dir; + struct ifreq req; + struct rte_ether_addr eth_addr; + int s; + + PMD_DRV_LOG(DEBUG, "%s: retry count %d", + __func__, hv->eal_hot_plug_retry); + + if (hv->eal_hot_plug_retry++ > NETVSC_MAX_HOTADD_RETRY) + return; + + snprintf(buf, sizeof(buf), "/sys/bus/pci/devices/%s/net", d->name); + di = opendir(buf); + if (!di) { + PMD_DRV_LOG(DEBUG, "%s: can't open directory %s, " + "retrying in 1 second", __func__, buf); + goto retry; + } + + while ((dir = readdir(di))) { + /* Skip . and .. directories */ + if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) + continue; + + /* trying to get mac address if this is a network device*/ + s = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); + if (s == -1) { + PMD_DRV_LOG(ERR, "Failed to create socket errno %d", + errno); + break; + } + strlcpy(req.ifr_name, dir->d_name, sizeof(req.ifr_name)); + ret = ioctl(s, SIOCGIFHWADDR, &req); + close(s); + if (ret == -1) { + PMD_DRV_LOG(ERR, + "Failed to send SIOCGIFHWADDR for device %s", + dir->d_name); + break; + } + if (req.ifr_hwaddr.sa_family != ARPHRD_ETHER) { + closedir(di); + return; + } + memcpy(eth_addr.addr_bytes, req.ifr_hwaddr.sa_data, + RTE_DIM(eth_addr.addr_bytes)); + + if (rte_is_same_ether_addr(ð_addr, dev->data->mac_addrs)) { + PMD_DRV_LOG(NOTICE, + "Found matching MAC address, adding device %s network name %s", + d->name, dir->d_name); + ret = rte_eal_hotplug_add(d->bus->name, d->name, + d->args); + if (ret) { + PMD_DRV_LOG(ERR, + "Failed to add PCI device %s", + d->name); + break; + } + } + /* When the code reaches here, we either have already added + * the device, or its MAC address did not match. + */ + closedir(di); + return; + } + closedir(di); +retry: + /* The device is still being initialized, retry after 1 second */ + rte_eal_alarm_set(1000000, netvsc_hotplug_retry, hv); +} -- 1.8.3.1
add meson support for FreeBSD OS Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- drivers/bus/vmbus/meson.build | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/bus/vmbus/meson.build b/drivers/bus/vmbus/meson.build index 60913d0..77f18ce 100644 --- a/drivers/bus/vmbus/meson.build +++ b/drivers/bus/vmbus/meson.build @@ -26,7 +26,11 @@ if is_linux sources += files('linux/vmbus_bus.c', 'linux/vmbus_uio.c') includes += include_directories('linux') +elif is_freebsd + sources += files('freebsd/vmbus_bus.c', + 'freebsd/vmbus_uio.c') + includes += include_directories('freebsd') else build = false - reason = 'only supported on Linux' + reason = 'only supported on Linux & FreeBSD' endif -- 1.8.3.1
updated MAINTAINERS and doc files for FreeBSD support Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com> Signed-off-by: Anand Thulasiram <avelu@juniper.net> --- MAINTAINERS | 2 ++ doc/guides/nics/netvsc.rst | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 7c4f541..01a494e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -567,6 +567,7 @@ F: app/test/test_vdev.c VMBUS bus driver M: Stephen Hemminger <sthemmin@microsoft.com> M: Long Li <longli@microsoft.com> +M: Srikanth Kaka <srikanth.k@oneconvergence.com> F: drivers/bus/vmbus/ @@ -823,6 +824,7 @@ F: doc/guides/nics/vdev_netvsc.rst Microsoft Hyper-V netvsc M: Stephen Hemminger <sthemmin@microsoft.com> M: Long Li <longli@microsoft.com> +M: Srikanth Kaka <srikanth.k@oneconvergence.com> F: drivers/net/netvsc/ F: doc/guides/nics/netvsc.rst F: doc/guides/nics/features/netvsc.ini diff --git a/doc/guides/nics/netvsc.rst b/doc/guides/nics/netvsc.rst index 77efe1d..12d1702 100644 --- a/doc/guides/nics/netvsc.rst +++ b/doc/guides/nics/netvsc.rst @@ -91,6 +91,12 @@ operations: The dpdk-devbind.py script can not be used since it only handles PCI devices. +On FreeBSD, with hv_uio kernel driver loaded, do the following: + + .. code-block:: console + + devctl set driver -f hn1 hv_uio + Prerequisites ------------- @@ -101,6 +107,11 @@ The following prerequisites apply: Full support of multiple queues requires the 4.17 kernel. It is possible to use the netvsc PMD with 4.16 kernel but it is limited to a single queue. +* FreeBSD support for UIO on vmbus is done with hv_uio driver and it is still + in `review`_ + +.. _`review`: https://reviews.freebsd.org/D32184 + Netvsc PMD arguments -------------------- -- 1.8.3.1
23/04/2022 06:28, Srikanth Kaka:
> This patchset requires FreeBSD VMBus kernel changes and
> HV_UIO driver. Both are currently under review at
> https://reviews.freebsd.org/D32184
What is the status of the dependencies on FreeBSD side?
Should we have another review of the DPDK series here?
18/05/2022 10:18, Thomas Monjalon:
> 23/04/2022 06:28, Srikanth Kaka:
> > This patchset requires FreeBSD VMBus kernel changes and
> > HV_UIO driver. Both are currently under review at
> > https://reviews.freebsd.org/D32184
>
> What is the status of the dependencies on FreeBSD side?
>
> Should we have another review of the DPDK series here?
No reply? No progress?
On Thu, 06 Oct 2022 16:58:01 +0200
Thomas Monjalon <thomas@monjalon.net> wrote:
> 18/05/2022 10:18, Thomas Monjalon:
> > 23/04/2022 06:28, Srikanth Kaka:
> > > This patchset requires FreeBSD VMBus kernel changes and
> > > HV_UIO driver. Both are currently under review at
> > > https://reviews.freebsd.org/D32184
> >
> > What is the status of the dependencies on FreeBSD side?
> >
> > Should we have another review of the DPDK series here?
>
> No reply? No progress?
Will this work with upstream FreeBSD?
Will it work with current DPDK 23.08?
Is someone going to maintain it and test it?
If so please resubmit a new version rebased on current tree.
Marking the patch series as "Awaiting Upstream"
On Sat, 23 Apr 2022 09:58:44 +0530
Srikanth Kaka <srikanth.k@oneconvergence.com> wrote:
> +/**
> + * Perform IOCTL to VMBUS device
> + *
> + * @param device
> + * A pointer to a rte_vmbus_device structure
> + * @param subchan
> + * Count of subchannels to open
> + */
> +__rte_experimental
> +int rte_vmbus_ioctl(struct rte_vmbus_device *device, uint32_t subchan);
The functionality is good, but have problem with naming.
The word 'ioctl' implies that this is a kernel system call but it is not.
Suggest rte_vmbus_add_channels() as function name instead.
On Sat, 23 Apr 2022 09:58:49 +0530
Srikanth Kaka <srikanth.k@oneconvergence.com> wrote:
> updated MAINTAINERS and doc files for FreeBSD support
>
> Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com>
> Signed-off-by: Vag Singh <vag.singh@oneconvergence.com>
> Signed-off-by: Anand Thulasiram <avelu@juniper.net>
> ---
> MAINTAINERS | 2 ++
> doc/guides/nics/netvsc.rst | 11 +++++++++++
> 2 files changed, 13 insertions(+)
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 7c4f541..01a494e 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -567,6 +567,7 @@ F: app/test/test_vdev.c
> VMBUS bus driver
> M: Stephen Hemminger <sthemmin@microsoft.com>
> M: Long Li <longli@microsoft.com>
> +M: Srikanth Kaka <srikanth.k@oneconvergence.com>
> F: drivers/bus/vmbus/
>
>
> @@ -823,6 +824,7 @@ F: doc/guides/nics/vdev_netvsc.rst
> Microsoft Hyper-V netvsc
> M: Stephen Hemminger <sthemmin@microsoft.com>
> M: Long Li <longli@microsoft.com>
> +M: Srikanth Kaka <srikanth.k@oneconvergence.com>
> F: drivers/net/netvsc/
> F: doc/guides/nics/netvsc.rst
> F: doc/guides/nics/features/netvsc.ini
> diff --git a/doc/guides/nics/netvsc.rst b/doc/guides/nics/netvsc.rst
> index 77efe1d..12d1702 100644
> --- a/doc/guides/nics/netvsc.rst
> +++ b/doc/guides/nics/netvsc.rst
> @@ -91,6 +91,12 @@ operations:
>
> The dpdk-devbind.py script can not be used since it only handles PCI devices.
>
> +On FreeBSD, with hv_uio kernel driver loaded, do the following:
> +
> + .. code-block:: console
> +
> + devctl set driver -f hn1 hv_uio
> +
>
> Prerequisites
> -------------
> @@ -101,6 +107,11 @@ The following prerequisites apply:
> Full support of multiple queues requires the 4.17 kernel. It is possible
> to use the netvsc PMD with 4.16 kernel but it is limited to a single queue.
>
> +* FreeBSD support for UIO on vmbus is done with hv_uio driver and it is still
> + in `review`_
> +
> +.. _`review`: https://reviews.freebsd.org/D32184
Looks like the FreeBSD UIO driver is still not merged upstream.
Any update on that?
For now, will leave the DPDK patches in patchwork (though they need to be rebased),
and mark them as "Awaiting upstream".
Alternatively, the BSD driver could be carried in DPDK.
Up to Bruce the FreeBSD maintainer to give feedback.