Re: [dpdk-dev] [PATCH 1/3] BSD: Support Port Hotplug function

["Iremonger, Bernard" <bernard.iremonger@intel.com>]

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Tetsuya Mukawa
> Sent: Wednesday, March 4, 2015 3:12 AM
> To: dev@dpdk.org
> Subject: [dpdk-dev] [PATCH 1/3] BSD: Support Port Hotplug function
> 
> This patch adds Hotplug support to BSD.
> 
> Signed-off-by: Tetsuya Mukawa <mukawa@igel.co.jp>
> ---
>  lib/librte_eal/bsdapp/eal/eal_pci.c           | 169 +++++++++++++++++++++++++-
>  lib/librte_eal/bsdapp/eal/rte_eal_version.map |   6 +
>  lib/librte_eal/common/include/rte_pci.h       |   1 +
>  lib/librte_ether/rte_ethdev.c                 |   1 +
>  4 files changed, 174 insertions(+), 3 deletions(-)
> 
> diff --git a/lib/librte_eal/bsdapp/eal/eal_pci.c b/lib/librte_eal/bsdapp/eal/eal_pci.c
> index 9193f80..fc5a088 100644
> --- a/lib/librte_eal/bsdapp/eal/eal_pci.c
> +++ b/lib/librte_eal/bsdapp/eal/eal_pci.c
> @@ -156,6 +156,26 @@ fail:
>  	return NULL;
>  }
> 
> +/* unmap a particular resource */
> +static int
> +pci_unmap_resource(void *requested_addr, size_t size) {
> +	if (requested_addr == NULL)
> +		return -EINVAL;
> +
> +	/* Unmap the PCI memory resource of device */
> +	if (munmap(requested_addr, size)) {
> +		RTE_LOG(ERR, EAL, "%s(): cannot munmap(%p, 0x%lx): %s\n",
> +				__func__, requested_addr, (unsigned long)size,
> +				strerror(errno));
> +	} else {
> +		RTE_LOG(DEBUG, EAL, "  PCI memory unmapped at %p\n",
> +				requested_addr);
> +		return -EFAULT;
> +	}
> +	return 0;
> +}
> +
>  static int
>  pci_uio_map_secondary(struct rte_pci_device *dev)  { @@ -270,6 +290,76 @@
> pci_uio_map_resource(struct rte_pci_device *dev)
>  	return (0);
>  }
> 
> +static int
> +pci_uio_unmap(struct uio_resource *uio_res) {
> +	int ret;
> +	unsigned i;
> +
> +	if (uio_res == NULL)
> +		return -EINVAL;
> +
> +	for (i = 0; i != uio_res->nb_maps; i++) {
> +		ret = pci_unmap_resource(uio_res->maps[i].addr,
> +				(size_t)uio_res->maps[i].size);
> +		if (ret < 0)
> +			return ret;
> +	}
> +	return 0;
> +}
> +
> +static struct uio_resource *
> +pci_uio_find_resource(struct rte_pci_device *dev) {
> +	struct uio_resource *uio_res;
> +
> +	if (dev == NULL)
> +		return NULL;
> +
> +	TAILQ_FOREACH(uio_res, uio_res_list, next) {
> +
> +		/* skip this element if it doesn't match our PCI address */
> +		if (!rte_eal_compare_pci_addr(&uio_res->pci_addr, &dev->addr))
> +			return uio_res;
> +	}
> +	return NULL;
> +}
> +
> +/* map the PCI resource of a PCI device in virtual memory */ static int
> +pci_uio_unmap_resource(struct rte_pci_device *dev) {
> +	struct uio_resource *uio_res;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	/* find an entry for the device */
> +	uio_res = pci_uio_find_resource(dev);
> +	if (uio_res == NULL)
> +		return -ENODEV;
> +
> +	/* secondary processes - just free maps */
> +	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> +		return pci_uio_unmap(uio_res);
> +
> +	TAILQ_REMOVE(uio_res_list, uio_res, next);
> +
> +	/* unmap all resources */
> +	pci_uio_unmap(uio_res);
> +
> +	/* free_uio resource */
> +	rte_free(uio_res);
> +
> +	/* close fd if in primary process */
> +	close(dev->intr_handle.fd);
> +
> +	dev->intr_handle.fd = -1;
> +	dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
> +
> +	return 0;
> +}
> +
>  /* Scan one pci sysfs entry, and fill the devices list from it. */  static int  pci_scan_one(int dev_pci_fd,
> struct pci_conf *conf) @@ -307,6 +397,9 @@ pci_scan_one(int dev_pci_fd, struct pci_conf *conf)
>  	/* FreeBSD has no NUMA support (yet) */
>  	dev->numa_node = 0;
> 
> +	/* FreeBSD has only one pass through driver */
> +	dev->pt_driver = RTE_PT_NIC_UIO;
> +
>  /* parse resources */
>  	switch (conf->pc_hdr & PCIM_HDRTYPE) {
>  	case PCIM_HDRTYPE_NORMAL:
> @@ -376,8 +469,8 @@ skipdev:
>   * Scan the content of the PCI bus, and add the devices in the devices
>   * list. Call pci_scan_one() for each pci entry found.
>   */
> -static int
> -pci_scan(void)
> +int
> +rte_eal_pci_scan(void)
>  {
>  	int fd = -1;
>  	unsigned dev_count = 0;
> @@ -487,6 +580,76 @@ rte_eal_pci_probe_one_driver(struct rte_pci_driver *dr, struct
> rte_pci_device *d
>  	return 1;
>  }
> 
> +/*
> + * If vendor/device ID match, call the devuninit() function of the
> + * driver.
> + */
> +int
> +rte_eal_pci_close_one_driver(struct rte_pci_driver *dr,
> +		struct rte_pci_device *dev)
> +{
> +	struct rte_pci_id *id_table;
> +	int ret;
> +
> +	if ((dr == NULL) || (dev == NULL))
> +		return -EINVAL;
> +
> +	for (id_table = dr->id_table ; id_table->vendor_id != 0; id_table++) {
> +
> +		/* check if device's identifiers match the driver's ones */
> +		if (id_table->vendor_id != dev->id.vendor_id &&
> +		    id_table->vendor_id != PCI_ANY_ID)
> +			continue;
> +		if (id_table->device_id != dev->id.device_id &&
> +		    id_table->device_id != PCI_ANY_ID)
> +			continue;
> +		if (id_table->subsystem_vendor_id !=
> +		    dev->id.subsystem_vendor_id &&
> +		    id_table->subsystem_vendor_id != PCI_ANY_ID)
> +			continue;
> +		if (id_table->subsystem_device_id !=
> +		    dev->id.subsystem_device_id &&
> +		    id_table->subsystem_device_id != PCI_ANY_ID)
> +			continue;
> +
> +		struct rte_pci_addr *loc = &dev->addr;
> +
> +		RTE_LOG(DEBUG, EAL,
> +			"PCI device "PCI_PRI_FMT" on NUMA socket %i\n",
> +			loc->domain, loc->bus, loc->devid, loc->function,
> +			dev->numa_node);
> +
> +		RTE_LOG(DEBUG, EAL,
> +			"  remove driver: %x:%x %s\n", dev->id.vendor_id,
> +			dev->id.device_id, dr->name);
> +
> +		/* no initialization when blacklisted, return without error */
> +		if (dev->devargs != NULL &&
> +			dev->devargs->type == RTE_DEVTYPE_BLACKLISTED_PCI) {
> +
> +			RTE_LOG(DEBUG, EAL,
> +				"  Device is blacklisted, not initializing\n");
> +			return 0;
> +		}
> +
> +		/* call the driver devuninit() function */
> +		if (dr->devuninit && (dr->devuninit(dev) < 0))
> +			return -1;	/* negative value is an error */
> +
> +		/* clear driver structure */
> +		dev->driver = NULL;
> +
> +		if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) {
> +			/* map resources for devices that use igb_uio */
> +			ret = pci_uio_unmap_resource(dev);
> +			if (ret != 0)
> +				return ret;
> +		}
> +	}
> +	/* return positive value if driver is not found */
> +	return 1;
> +}
> +
Hi Tetsuya,

The above functions being added to /lib/librte_eal/bsdapp/eal/eal_pci.c are very similar to the functions
in lib/librte_eal/linuxapp/eal/eal_pci.c the main difference seems to be "struct uio_resource *uio_res" in bsd
and "struct mapped_pci_resource *uio_res " in linux.
Both of these structures are the same although named differently.
There are at present only 3 references to   "struct uio_resource" in librte_eal/bsdapp/eal/eal_pci.c. If "struct uio_resource"  was renamed to" struct mapped_pci_resource "  the functions being  added to bsdapp/eal/eal_pci.c would be the same as the functions in linuxapp/eal/eal_pci.c.
If both groups of functions were the same  it would make sense to have only one copy of these functions  which could be moved into librte_eal/common/eal_common_pci.c .

There may be some extra work in doing  this but I think it would make the code more maintainable in the  future.
What do you think?

Regards,

Bernard.


>  /* Init the PCI EAL subsystem */
>  int
>  rte_eal_pci_init(void)
> @@ -499,7 +662,7 @@ rte_eal_pci_init(void)
>  	if (internal_config.no_pci)
>  		return 0;
> 
> -	if (pci_scan() < 0) {
> +	if (rte_eal_pci_scan() < 0) {
>  		RTE_LOG(ERR, EAL, "%s(): Cannot scan PCI bus\n", __func__);
>  		return -1;
>  	}
> diff --git a/lib/librte_eal/bsdapp/eal/rte_eal_version.map
> b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
> index d83524d..54f5ff1 100644
> --- a/lib/librte_eal/bsdapp/eal/rte_eal_version.map
> +++ b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
> @@ -37,9 +37,13 @@ DPDK_2.0 {
>  	rte_eal_lcore_role;
>  	rte_eal_mp_remote_launch;
>  	rte_eal_mp_wait_lcore;
> +	rte_eal_parse_devargs_str;
> +	rte_eal_pci_close_one;
>  	rte_eal_pci_dump;
>  	rte_eal_pci_probe;
> +	rte_eal_pci_probe_one;
>  	rte_eal_pci_register;
> +	rte_eal_pci_scan;
>  	rte_eal_pci_unregister;
>  	rte_eal_process_type;
>  	rte_eal_remote_launch;
> @@ -47,6 +51,8 @@ DPDK_2.0 {
>  	rte_eal_tailq_lookup_by_idx;
>  	rte_eal_tailq_reserve;
>  	rte_eal_tailq_reserve_by_idx;
> +	rte_eal_vdev_init;
> +	rte_eal_vdev_uninit;
>  	rte_eal_wait_lcore;
>  	rte_exit;
>  	rte_get_hpet_cycles;
> diff --git a/lib/librte_eal/common/include/rte_pci.h b/lib/librte_eal/common/include/rte_pci.h
> index b9cdf8b..dc74821 100644
> --- a/lib/librte_eal/common/include/rte_pci.h
> +++ b/lib/librte_eal/common/include/rte_pci.h
> @@ -147,6 +147,7 @@ enum rte_pt_driver {
>  	RTE_PT_IGB_UIO		= 1,
>  	RTE_PT_VFIO		= 2,
>  	RTE_PT_UIO_GENERIC	= 3,
> +	RTE_PT_NIC_UIO		= 4, /* Driver for BSD */
>  };
> 
>  /**
> diff --git a/lib/librte_ether/rte_ethdev.c b/lib/librte_ether/rte_ethdev.c index 6ea7a17..9687fab
> 100644
> --- a/lib/librte_ether/rte_ethdev.c
> +++ b/lib/librte_ether/rte_ethdev.c
> @@ -516,6 +516,7 @@ rte_eth_dev_is_detachable(uint8_t port_id)
>  		switch (rte_eth_devices[port_id].pci_dev->pt_driver) {
>  		case RTE_PT_IGB_UIO:
>  		case RTE_PT_UIO_GENERIC:
> +		case RTE_PT_NIC_UIO:
>  			break;
>  		case RTE_PT_VFIO:
>  		default:
> --
> 1.9.1