Re: [dpdk-dev] [PATCH v3 2/4] net/vhost: delay vhost driver setup

[Maxime Coquelin <maxime.coquelin@redhat.com>]

Thanks Yinan for reporting the issue.

I'm working on a fix, which should be ready today:
1. Don't do vhost setup if device is already in the list
2. Fix the error path in setup function.

Maxime
On 2/18/20 9:50 AM, Wang, Yinan wrote:
> Hi Itsuro/Maxime,
> 
> Below patch (commit id:3d01b759d2679c216725689eabe44147d1737326)blocked vhost/virtio basic test: If re-config vhost port，testpmd will launch failed.
> 
> Reproduce steps:
> 
> ./x86_64-native-linuxapp-gcc/app/testpmd -l 2-4 -n 4 --socket-mem 1024,1024  --legacy-mem --file-prefix=vhost --vdev 'net_vhost0,iface=vhost-net,queues=1,client=0' --no-pci -- -i --txd=1024 --rxd=1024
>     testpmd>set fwd csum
>     testpmd>stop
>     testpmd>port stop 0
>     testpmd>csum set tcp sw 0
>     testpmd>port start 0
> 
> VHOST_CONFIG: vhost-user server: socket created, fd: 26
> VHOST_CONFIG: failed to bind to vhost-net: Address already in use; remove it and try again
> Failed to start driver for vhost-net
> Port0 dev_configure = -1
> Fail to configure port 0
> 
> BR,
> Yinan
>> -----Original Message-----
>> From: dev <dev-bounces@dpdk.org> On Behalf Of Itsuro Oda
>> Sent: 2020年2月6日 9:40
>> To: dev@dpdk.org; maxime.coquelin@redhat.com; Bie, Tiwei
>> <tiwei.bie@intel.com>; Wang, Zhihong <zhihong.wang@intel.com>; Burakov,
>> Anatoly <anatoly.burakov@intel.com>
>> Cc: stable@dpdk.org
>> Subject: [dpdk-dev] [PATCH v3 2/4] net/vhost: delay vhost driver setup
>>
>> Vhost driver setup is delayed at eth_dev configuration in order to be able to set
>> it from a secondary process.
>>
>> Fixes: 4852aa8f6e21 (drivers/net: enable hotplug on secondary process)
>> Cc: stable@dpdk.org
>>
>> Signed-off-by: Itsuro Oda <oda@valinux.co.jp>
>> Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
>> ---
>>  drivers/net/vhost/rte_eth_vhost.c | 130 ++++++++++++++++++------------
>>  1 file changed, 78 insertions(+), 52 deletions(-)
>>
>> diff --git a/drivers/net/vhost/rte_eth_vhost.c
>> b/drivers/net/vhost/rte_eth_vhost.c
>> index cea2ead2d..d7bba5c6e 100644
>> --- a/drivers/net/vhost/rte_eth_vhost.c
>> +++ b/drivers/net/vhost/rte_eth_vhost.c
>> @@ -97,6 +97,8 @@ struct pmd_internal {
>>  	rte_atomic32_t dev_attached;
>>  	char *dev_name;
>>  	char *iface_name;
>> +	uint64_t flags;
>> +	uint64_t disable_flags;
>>  	uint16_t max_queues;
>>  	int vid;
>>  	rte_atomic32_t started;
>> @@ -491,17 +493,6 @@ eth_vhost_tx(void *q, struct rte_mbuf **bufs,
>> uint16_t nb_bufs)
>>  	return nb_tx;
>>  }
>>
>> -static int
>> -eth_dev_configure(struct rte_eth_dev *dev __rte_unused) -{
>> -	struct pmd_internal *internal = dev->data->dev_private;
>> -	const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
>> -
>> -	internal->vlan_strip = !!(rxmode->offloads &
>> DEV_RX_OFFLOAD_VLAN_STRIP);
>> -
>> -	return 0;
>> -}
>> -
>>  static inline struct internal_list *
>>  find_internal_resource(char *ifname)
>>  {
>> @@ -877,6 +868,62 @@ static struct vhost_device_ops vhost_ops = {
>>  	.vring_state_changed = vring_state_changed,  };
>>
>> +static int
>> +vhost_driver_setup(struct rte_eth_dev *eth_dev) {
>> +	struct pmd_internal *internal = eth_dev->data->dev_private;
>> +	struct internal_list *list = NULL;
>> +	struct rte_vhost_vring_state *vring_state = NULL;
>> +	unsigned int numa_node = eth_dev->device->numa_node;
>> +	const char *name = eth_dev->device->name;
>> +
>> +	list = rte_zmalloc_socket(name, sizeof(*list), 0, numa_node);
>> +	if (list == NULL)
>> +		goto error;
>> +
>> +	vring_state = rte_zmalloc_socket(name, sizeof(*vring_state),
>> +					 0, numa_node);
>> +	if (vring_state == NULL)
>> +		goto error;
>> +
>> +	list->eth_dev = eth_dev;
>> +	pthread_mutex_lock(&internal_list_lock);
>> +	TAILQ_INSERT_TAIL(&internal_list, list, next);
>> +	pthread_mutex_unlock(&internal_list_lock);
>> +
>> +	rte_spinlock_init(&vring_state->lock);
>> +	vring_states[eth_dev->data->port_id] = vring_state;
>> +
>> +	if (rte_vhost_driver_register(internal->iface_name, internal->flags))
>> +		goto error;
>> +
>> +	if (internal->disable_flags) {
>> +		if (rte_vhost_driver_disable_features(internal->iface_name,
>> +						      internal->disable_flags))
>> +			goto error;
>> +	}
>> +
>> +	if (rte_vhost_driver_callback_register(internal->iface_name,
>> +					       &vhost_ops) < 0) {
>> +		VHOST_LOG(ERR, "Can't register callbacks\n");
>> +		goto error;
>> +	}
>> +
>> +	if (rte_vhost_driver_start(internal->iface_name) < 0) {
>> +		VHOST_LOG(ERR, "Failed to start driver for %s\n",
>> +			  internal->iface_name);
>> +		goto error;
>> +	}
>> +
>> +	return 0;
>> +
>> +error:
>> +	rte_free(vring_state);
>> +	rte_free(list);
>> +
>> +	return -1;
>> +}
>> +
>>  int
>>  rte_eth_vhost_get_queue_event(uint16_t port_id,
>>  		struct rte_eth_vhost_queue_event *event) @@ -943,6 +990,24 @@
>> rte_eth_vhost_get_vid_from_port_id(uint16_t port_id)
>>  	return vid;
>>  }
>>
>> +static int
>> +eth_dev_configure(struct rte_eth_dev *dev) {
>> +	struct pmd_internal *internal = dev->data->dev_private;
>> +	const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
>> +
>> +	/* NOTE: the same process has to operate a vhost interface
>> +	 * from beginning to end (from eth_dev configure to eth_dev close).
>> +	 * It is user's responsibility at the moment.
>> +	 */
>> +	if (vhost_driver_setup(dev) < 0)
>> +		return -1;
>> +
>> +	internal->vlan_strip = !!(rxmode->offloads &
>> +DEV_RX_OFFLOAD_VLAN_STRIP);
>> +
>> +	return 0;
>> +}
>> +
>>  static int
>>  eth_dev_start(struct rte_eth_dev *eth_dev)  { @@ -1219,16 +1284,10 @@
>> eth_dev_vhost_create(struct rte_vdev_device *dev, char *iface_name,
>>  	struct pmd_internal *internal = NULL;
>>  	struct rte_eth_dev *eth_dev = NULL;
>>  	struct rte_ether_addr *eth_addr = NULL;
>> -	struct rte_vhost_vring_state *vring_state = NULL;
>> -	struct internal_list *list = NULL;
>>
>>  	VHOST_LOG(INFO, "Creating VHOST-USER backend on numa
>> socket %u\n",
>>  		numa_node);
>>
>> -	list = rte_zmalloc_socket(name, sizeof(*list), 0, numa_node);
>> -	if (list == NULL)
>> -		goto error;
>> -
>>  	/* reserve an ethdev entry */
>>  	eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internal));
>>  	if (eth_dev == NULL)
>> @@ -1242,11 +1301,6 @@ eth_dev_vhost_create(struct rte_vdev_device *dev,
>> char *iface_name,
>>  	*eth_addr = base_eth_addr;
>>  	eth_addr->addr_bytes[5] = eth_dev->data->port_id;
>>
>> -	vring_state = rte_zmalloc_socket(name,
>> -			sizeof(*vring_state), 0, numa_node);
>> -	if (vring_state == NULL)
>> -		goto error;
>> -
>>  	/* now put it all together
>>  	 * - store queue data in internal,
>>  	 * - point eth_dev_data to internals
>> @@ -1262,18 +1316,12 @@ eth_dev_vhost_create(struct rte_vdev_device
>> *dev, char *iface_name,
>>  		goto error;
>>  	strcpy(internal->iface_name, iface_name);
>>
>> -	list->eth_dev = eth_dev;
>> -	pthread_mutex_lock(&internal_list_lock);
>> -	TAILQ_INSERT_TAIL(&internal_list, list, next);
>> -	pthread_mutex_unlock(&internal_list_lock);
>> -
>> -	rte_spinlock_init(&vring_state->lock);
>> -	vring_states[eth_dev->data->port_id] = vring_state;
>> -
>>  	data->nb_rx_queues = queues;
>>  	data->nb_tx_queues = queues;
>>  	internal->max_queues = queues;
>>  	internal->vid = -1;
>> +	internal->flags = flags;
>> +	internal->disable_flags = disable_flags;
>>  	data->dev_link = pmd_link;
>>  	data->dev_flags = RTE_ETH_DEV_INTR_LSC |
>> RTE_ETH_DEV_CLOSE_REMOVE;
>>
>> @@ -1283,26 +1331,6 @@ eth_dev_vhost_create(struct rte_vdev_device *dev,
>> char *iface_name,
>>  	eth_dev->rx_pkt_burst = eth_vhost_rx;
>>  	eth_dev->tx_pkt_burst = eth_vhost_tx;
>>
>> -	if (rte_vhost_driver_register(iface_name, flags))
>> -		goto error;
>> -
>> -	if (disable_flags) {
>> -		if (rte_vhost_driver_disable_features(iface_name,
>> -					disable_flags))
>> -			goto error;
>> -	}
>> -
>> -	if (rte_vhost_driver_callback_register(iface_name, &vhost_ops) < 0) {
>> -		VHOST_LOG(ERR, "Can't register callbacks\n");
>> -		goto error;
>> -	}
>> -
>> -	if (rte_vhost_driver_start(iface_name) < 0) {
>> -		VHOST_LOG(ERR, "Failed to start driver for %s\n",
>> -			iface_name);
>> -		goto error;
>> -	}
>> -
>>  	rte_eth_dev_probing_finish(eth_dev);
>>  	return 0;
>>
>> @@ -1311,9 +1339,7 @@ eth_dev_vhost_create(struct rte_vdev_device *dev,
>> char *iface_name,
>>  		rte_free(internal->iface_name);
>>  		free(internal->dev_name);
>>  	}
>> -	rte_free(vring_state);
>>  	rte_eth_dev_release_port(eth_dev);
>> -	rte_free(list);
>>
>>  	return -1;
>>  }
>> --
>> 2.17.0
>