Re: [dpdk-dev] [PATCH v2 04/14] vhost: make vDPA framework bus agnostic

[Adrian Moreno <amorenoz@redhat.com>]

On 6/24/20 2:26 PM, Maxime Coquelin wrote:
> This patch makes the vDPA framework to no more
> support only PCI devices, but any devices by relying
> on the generic device name as identifier.
> 
> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> ---
>  drivers/vdpa/ifc/ifcvf_vdpa.c          |  9 +--
>  drivers/vdpa/mlx5/mlx5_vdpa.c          |  8 +--
>  drivers/vdpa/mlx5/mlx5_vdpa.h          |  2 +-
>  examples/vdpa/main.c                   | 49 ++++++++--------
>  lib/librte_vhost/rte_vdpa.h            | 42 +++++++-------
>  lib/librte_vhost/rte_vhost_version.map |  1 +
>  lib/librte_vhost/vdpa.c                | 79 +++++++++++---------------
>  7 files changed, 87 insertions(+), 103 deletions(-)
> 
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index ec97178dcb..ac9e218c23 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -47,7 +47,6 @@ static const char * const ifcvf_valid_arguments[] = {
>  static int ifcvf_vdpa_logtype;
>  
>  struct ifcvf_internal {
> -	struct rte_vdpa_dev_addr dev_addr;
>  	struct rte_pci_device *pdev;
>  	struct ifcvf_hw hw;
>  	int vfio_container_fd;
> @@ -116,7 +115,8 @@ find_internal_resource_by_dev(struct rte_pci_device *pdev)
>  	pthread_mutex_lock(&internal_list_lock);
>  
>  	TAILQ_FOREACH(list, &internal_list, next) {
> -		if (pdev == list->internal->pdev) {
> +		if (!rte_pci_addr_cmp(&pdev->addr,
> +					&list->internal->pdev->addr)) {
>  			found = 1;
>  			break;
>  		}
> @@ -1176,8 +1176,6 @@ ifcvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
>  		(1ULL << VHOST_USER_F_PROTOCOL_FEATURES) |
>  		(1ULL << VHOST_F_LOG_ALL);
>  
> -	internal->dev_addr.pci_addr = pci_dev->addr;
> -	internal->dev_addr.type = VDPA_ADDR_PCI;
>  	list->internal = internal;
>  
>  	if (rte_kvargs_count(kvlist, IFCVF_SW_FALLBACK_LM)) {
> @@ -1188,8 +1186,7 @@ ifcvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
>  	}
>  	internal->sw_lm = sw_fallback_lm;
>  
> -	internal->did = rte_vdpa_register_device(&internal->dev_addr,
> -				&ifcvf_ops);
> +	internal->did = rte_vdpa_register_device(&pci_dev->device, &ifcvf_ops);
>  	if (internal->did < 0) {
>  		DRV_LOG(ERR, "failed to register device %s", pci_dev->name);
>  		goto error;
> diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
> index 8b0b3b8193..7b5ae62bdc 100644
> --- a/drivers/vdpa/mlx5/mlx5_vdpa.c
> +++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
> @@ -681,14 +681,13 @@ mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
>  	priv->caps = attr.vdpa;
>  	priv->log_max_rqt_size = attr.log_max_rqt_size;
>  	priv->ctx = ctx;
> -	priv->dev_addr.pci_addr = pci_dev->addr;
> -	priv->dev_addr.type = VDPA_ADDR_PCI;
> +	priv->pci_dev = pci_dev;
>  	priv->var = mlx5_glue->dv_alloc_var(ctx, 0);
>  	if (!priv->var) {
>  		DRV_LOG(ERR, "Failed to allocate VAR %u.\n", errno);
>  		goto error;
>  	}
> -	priv->id = rte_vdpa_register_device(&priv->dev_addr, &mlx5_vdpa_ops);
> +	priv->id = rte_vdpa_register_device(&pci_dev->device, &mlx5_vdpa_ops);
>  	if (priv->id < 0) {
>  		DRV_LOG(ERR, "Failed to register vDPA device.");
>  		rte_errno = rte_errno ? rte_errno : EINVAL;
> @@ -730,8 +729,7 @@ mlx5_vdpa_pci_remove(struct rte_pci_device *pci_dev)
>  
>  	pthread_mutex_lock(&priv_list_lock);
>  	TAILQ_FOREACH(priv, &priv_list, next) {
> -		if (memcmp(&priv->dev_addr.pci_addr, &pci_dev->addr,
> -			   sizeof(pci_dev->addr)) == 0) {
> +		if (!rte_pci_addr_cmp(&priv->pci_dev->addr, &pci_dev->addr)) {
>  			found = 1;
>  			break;
>  		}
> diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h
> index 80b4c4bda9..5fc801eff3 100644
> --- a/drivers/vdpa/mlx5/mlx5_vdpa.h
> +++ b/drivers/vdpa/mlx5/mlx5_vdpa.h
> @@ -106,7 +106,7 @@ struct mlx5_vdpa_priv {
>  	int id; /* vDPA device id. */
>  	int vid; /* vhost device id. */
>  	struct ibv_context *ctx; /* Device context. */
> -	struct rte_vdpa_dev_addr dev_addr;
> +	struct rte_pci_device *pci_dev;
>  	struct mlx5_hca_vdpa_attr caps;
>  	uint32_t pdn; /* Protection Domain number. */
>  	struct ibv_pd *pd;
> diff --git a/examples/vdpa/main.c b/examples/vdpa/main.c
> index bb30796df6..e72f6646e7 100644
> --- a/examples/vdpa/main.c
> +++ b/examples/vdpa/main.c
> @@ -280,10 +280,14 @@ static void cmd_list_vdpa_devices_parsed(
>  	uint32_t queue_num;
>  	uint64_t features;
>  	struct rte_vdpa_device *vdev;
> -	struct rte_pci_addr addr;
> +	struct rte_device *dev;
> +	struct rte_dev_iterator dev_iter;
>  
> -	cmdline_printf(cl, "device id\tdevice address\tqueue num\tsupported features\n");
> -	for (did = 0; did < dev_total; did++) {
> +	cmdline_printf(cl, "device id\tdevice name\tqueue num\tsupported features\n");
> +	RTE_DEV_FOREACH(dev, "class=vdpa", &dev_iter) {
> +		did = rte_vdpa_find_device_id_by_name(dev->name);
> +		if (did < 0)
> +			continue;
>  		vdev = rte_vdpa_get_device(did);
>  		if (!vdev)
>  			continue;
> @@ -299,11 +303,8 @@ static void cmd_list_vdpa_devices_parsed(
>  				"for device id %d.\n", did);
>  			continue;
>  		}
> -		addr = vdev->addr.pci_addr;
> -		cmdline_printf(cl,
> -			"%d\t\t" PCI_PRI_FMT "\t%" PRIu32 "\t\t0x%" PRIx64 "\n",
> -			did, addr.domain, addr.bus, addr.devid,
> -			addr.function, queue_num, features);
> +		cmdline_printf(cl, "%d\t\t%s\t\t%" PRIu32 "\t\t0x%" PRIx64 "\n",
> +			did, dev->name, queue_num, features);
>  	}
>  }
>  
> @@ -333,17 +334,12 @@ static void cmd_create_vdpa_port_parsed(void *parsed_result,
>  {
>  	int did;
>  	struct cmd_create_result *res = parsed_result;
> -	struct rte_vdpa_dev_addr addr;
>  
>  	rte_strscpy(vports[devcnt].ifname, res->socket_path, MAX_PATH_LEN);
> -	if (rte_pci_addr_parse(res->bdf, &addr.pci_addr) != 0) {
> -		cmdline_printf(cl, "Unable to parse the given bdf.\n");
> -		return;
> -	}
> -	addr.type = VDPA_ADDR_PCI;
> -	did = rte_vdpa_find_device_id(&addr);
> +	did = rte_vdpa_find_device_id_by_name(res->bdf);
>  	if (did < 0) {
> -		cmdline_printf(cl, "Unable to find vdpa device id.\n");
> +		cmdline_printf(cl, "Unable to find vdpa device id for %s.\n",
> +				res->bdf);
>  		return;
>  	}
>  
> @@ -519,9 +515,11 @@ int
>  main(int argc, char *argv[])
>  {
>  	char ch;
> -	int i;
> +	int did;
>  	int ret;
>  	struct cmdline *cl;
> +	struct rte_device *dev;
> +	struct rte_dev_iterator dev_iter;
>  
>  	ret = rte_eal_init(argc, argv);
>  	if (ret < 0)
> @@ -547,13 +545,18 @@ main(int argc, char *argv[])
>  		cmdline_interact(cl);
>  		cmdline_stdin_exit(cl);
>  	} else {
> -		for (i = 0; i < RTE_MIN(MAX_VDPA_SAMPLE_PORTS, dev_total);
> -				i++) {
> -			vports[i].did = i;
> -			snprintf(vports[i].ifname, MAX_PATH_LEN, "%s%d",
> -					iface, i);
> +		RTE_DEV_FOREACH(dev, "class=vdpa", &dev_iter) {
> +			did = rte_vdpa_find_device_id_by_name(dev->name);
> +			if (did < 0) {
> +				rte_panic("Failed to find device id for %s\n",
> +						dev->name);
> +			}
> +			vports[devcnt].did = did;
> +			snprintf(vports[devcnt].ifname, MAX_PATH_LEN, "%s%d",
> +					iface, devcnt);
>  
> -			start_vdpa(&vports[i]);
> +			start_vdpa(&vports[devcnt]);
> +			devcnt++;
>  		}
>  
>  		printf("enter \'q\' to quit\n");
> diff --git a/lib/librte_vhost/rte_vdpa.h b/lib/librte_vhost/rte_vdpa.h
> index ecb3d911d0..b752dfeb96 100644
> --- a/lib/librte_vhost/rte_vdpa.h
> +++ b/lib/librte_vhost/rte_vdpa.h
> @@ -18,25 +18,6 @@
>  
>  #define MAX_VDPA_NAME_LEN 128
>  
> -enum vdpa_addr_type {
> -	VDPA_ADDR_PCI,
> -	VDPA_ADDR_MAX
> -};
> -
> -/**
> - * vdpa device address
> - */
> -struct rte_vdpa_dev_addr {
> -	/** vdpa address type */
> -	enum vdpa_addr_type type;
> -
> -	/** vdpa pci address */
> -	union {
> -		uint8_t __dummy[64];
> -		struct rte_pci_addr pci_addr;
> -	};
> -};
> -
>  /** Maximum name length for statistics counters */
>  #define RTE_VDPA_STATS_NAME_SIZE 64
>  
> @@ -120,8 +101,8 @@ struct rte_vdpa_dev_ops {
>   * vdpa device structure includes device address and device operations.
>   */
>  struct rte_vdpa_device {
> -	/** vdpa device address */
> -	struct rte_vdpa_dev_addr addr;
> +	/** Generic device information */
> +	struct rte_device *device;
>  	/** vdpa device operations */
>  	struct rte_vdpa_dev_ops *ops;
>  } __rte_cache_aligned;
> @@ -141,7 +122,7 @@ struct rte_vdpa_device {
>   */
>  __rte_experimental
>  int
> -rte_vdpa_register_device(struct rte_vdpa_dev_addr *addr,
> +rte_vdpa_register_device(struct rte_device *rte_dev,
>  		struct rte_vdpa_dev_ops *ops);
>  
>  /**
> @@ -159,6 +140,21 @@ __rte_experimental
>  int
>  rte_vdpa_unregister_device(int did);
>  
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Find the device id of a vdpa device from its name
> + *
> + * @param name
> + *  the vdpa device name
> + * @return
> + *  device id on success, -1 on failure
> + */
> +__rte_experimental
> +int
> +rte_vdpa_find_device_id_by_name(const char *name);
> +
>  /**
>   * @warning
>   * @b EXPERIMENTAL: this API may change without prior notice
> @@ -172,7 +168,7 @@ rte_vdpa_unregister_device(int did);
>   */
>  __rte_experimental
>  int
> -rte_vdpa_find_device_id(struct rte_vdpa_dev_addr *addr);
> +rte_vdpa_find_device_id(struct rte_vdpa_device *dev);
>  
>  /**
>   * @warning
> diff --git a/lib/librte_vhost/rte_vhost_version.map b/lib/librte_vhost/rte_vhost_version.map
> index 3a2f7df1e5..9325ee4227 100644
> --- a/lib/librte_vhost/rte_vhost_version.map
> +++ b/lib/librte_vhost/rte_vhost_version.map
> @@ -69,4 +69,5 @@ EXPERIMENTAL {
>  	rte_vhost_get_vhost_ring_inflight;
>  	rte_vhost_get_vring_base_from_inflight;
>  	rte_vhost_slave_config_change;
> +	rte_vdpa_find_device_id_by_name;
>  };
> diff --git a/lib/librte_vhost/vdpa.c b/lib/librte_vhost/vdpa.c
> index a429841691..07db5b9e46 100644
> --- a/lib/librte_vhost/vdpa.c
> +++ b/lib/librte_vhost/vdpa.c
> @@ -18,43 +18,22 @@
>  static struct rte_vdpa_device vdpa_devices[MAX_VHOST_DEVICE];
>  static uint32_t vdpa_device_num;
>  
> -static bool
> -is_same_vdpa_device(struct rte_vdpa_dev_addr *a,
> -		struct rte_vdpa_dev_addr *b)
> -{
> -	bool ret = true;
> -
> -	if (a->type != b->type)
> -		return false;
> -
> -	switch (a->type) {
> -	case VDPA_ADDR_PCI:
> -		if (a->pci_addr.domain != b->pci_addr.domain ||
> -				a->pci_addr.bus != b->pci_addr.bus ||
> -				a->pci_addr.devid != b->pci_addr.devid ||
> -				a->pci_addr.function != b->pci_addr.function)
> -			ret = false;
> -		break;
> -	default:
> -		break;
> -	}
> -
> -	return ret;
> -}
> -
>  int
> -rte_vdpa_register_device(struct rte_vdpa_dev_addr *addr,
> +rte_vdpa_register_device(struct rte_device *rte_dev,
>  		struct rte_vdpa_dev_ops *ops)
>  {
>  	struct rte_vdpa_device *dev;
>  	int i;
>  
> -	if (vdpa_device_num >= MAX_VHOST_DEVICE || addr == NULL || ops == NULL)
> +	if (vdpa_device_num >= MAX_VHOST_DEVICE || ops == NULL)
>  		return -1;
>  
>  	for (i = 0; i < MAX_VHOST_DEVICE; i++) {
>  		dev = &vdpa_devices[i];
> -		if (dev->ops && is_same_vdpa_device(&dev->addr, addr))
> +		if (dev->ops == NULL)
> +			continue;
> +
> +		if (dev->device == rte_dev)
>  			return -1;
>  	}
>  
> @@ -67,7 +46,7 @@ rte_vdpa_register_device(struct rte_vdpa_dev_addr *addr,
>  		return -1;
>  
>  	dev = &vdpa_devices[i];
> -	memcpy(&dev->addr, addr, sizeof(struct rte_vdpa_dev_addr));
> +	dev->device = rte_dev;
>  	dev->ops = ops;
>  	vdpa_device_num++;
>  
> @@ -87,12 +66,33 @@ rte_vdpa_unregister_device(int did)
>  }
>  
>  int
> -rte_vdpa_find_device_id(struct rte_vdpa_dev_addr *addr)
> +rte_vdpa_find_device_id(struct rte_vdpa_device *dev)
> +{
> +	struct rte_vdpa_device *tmp_dev;
> +	int i;
> +
> +	if (dev == NULL)
> +		return -1;
> +
> +	for (i = 0; i < MAX_VHOST_DEVICE; ++i) {
> +		tmp_dev = &vdpa_devices[i];
> +		if (tmp_dev->ops == NULL)
> +			continue;
> +
> +		if (tmp_dev == dev)
> +			return i;
> +	}
> +
> +	return -1;
> +}
> +
> +int
> +rte_vdpa_find_device_id_by_name(const char *name)
>  {
>  	struct rte_vdpa_device *dev;
>  	int i;
>  
> -	if (addr == NULL)
> +	if (name == NULL)
>  		return -1;
>  
>  	for (i = 0; i < MAX_VHOST_DEVICE; ++i) {
> @@ -100,7 +100,7 @@ rte_vdpa_find_device_id(struct rte_vdpa_dev_addr *addr)
>  		if (dev->ops == NULL)
>  			continue;
>  
> -		if (is_same_vdpa_device(&dev->addr, addr))
> +		if (strcmp(dev->device->name, name) == 0)
>  			return i;
>  	}
Considering this function is exposed to the API, I'd consider using
strncmp(dev->device->name, name, RTE_DEV_NAME_MAX_LEN)


>  
> @@ -288,21 +288,10 @@ static int
>  vdpa_dev_match(struct rte_vdpa_device *dev,
>  	      const struct rte_device *rte_dev)
>  {
> -	struct rte_vdpa_dev_addr addr;
> +	if (dev->device == rte_dev)
> +		return 0;
>  
> -	/*  Only PCI bus supported for now */
> -	if (strcmp(rte_dev->bus->name, "pci") != 0)
> -		return -1;
> -
> -	addr.type = VDPA_ADDR_PCI;
> -
> -	if (rte_pci_addr_parse(rte_dev->name, &addr.pci_addr) != 0)
> -		return -1;
> -
> -	if (!is_same_vdpa_device(&dev->addr, &addr))
> -		return -1;
> -
> -	return 0;
> +	return -1;
>  }
>  
>  /* Generic rte_vdpa_dev comparison function. */
> 

-- 
Adrián Moreno