Re: [dpdk-dev] [PATCH v2 3/4] net/failsafe: replace local sub-device with shared data

["Gaëtan Rivet" <gaetan.rivet@6wind.com>]

On Tue, Mar 05, 2019 at 09:52:05AM +0000, Raslan Darawsheh wrote:
> In multiprocess context, the pointer to sub-device is shared between
> processes. Previously, it was a pointer to per process eth_dev so
> it's needed to replace this dependency.
> 
> Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
> Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
> ---
> v2: - moved comment in fs_sdev about subs to this commit
>     - added parenthesis around macro arguments.
> ---
>  drivers/net/failsafe/failsafe_eal.c     |  2 +-
>  drivers/net/failsafe/failsafe_ether.c   |  7 ++++---
>  drivers/net/failsafe/failsafe_private.h | 13 ++++++++-----
>  3 files changed, 13 insertions(+), 9 deletions(-)
> 
> diff --git a/drivers/net/failsafe/failsafe_eal.c b/drivers/net/failsafe/failsafe_eal.c
> index 56d1669..6fac4b6 100644
> --- a/drivers/net/failsafe/failsafe_eal.c
> +++ b/drivers/net/failsafe/failsafe_eal.c
> @@ -112,7 +112,7 @@ fs_bus_init(struct rte_eth_dev *dev)
>  				continue;
>  			}
>  		}
> -		ETH(sdev) = &rte_eth_devices[pid];
> +		sdev->data = rte_eth_devices[pid].data;
>  		SUB_ID(sdev) = i;
>  		sdev->fs_port_id = dev->data->port_id;
>  		sdev->dev = ETH(sdev)->device;
> diff --git a/drivers/net/failsafe/failsafe_ether.c b/drivers/net/failsafe/failsafe_ether.c
> index d5b1488..e1fff59 100644
> --- a/drivers/net/failsafe/failsafe_ether.c
> +++ b/drivers/net/failsafe/failsafe_ether.c
> @@ -267,18 +267,19 @@ static void
>  fs_dev_remove(struct sub_device *sdev)
>  {
>  	int ret;
> +	struct rte_eth_dev *edev = ETH(sdev);

I'd have added that above the "int ret;".
(inverse christmas tree and all that.)

>  
>  	if (sdev == NULL)
>  		return;
>  	switch (sdev->state) {
>  	case DEV_STARTED:
>  		failsafe_rx_intr_uninstall_subdevice(sdev);
> -		rte_eth_dev_stop(PORT_ID(sdev));
> +		rte_eth_dev_stop(edev->data->port_id);
>  		sdev->state = DEV_ACTIVE;
>  		/* fallthrough */
>  	case DEV_ACTIVE:
>  		failsafe_eth_dev_unregister_callbacks(sdev);
> -		rte_eth_dev_close(PORT_ID(sdev));
> +		rte_eth_dev_close(edev->data->port_id);

Ok I see. I missed that during the first reading, the private_data is
zeroed on dev_close(), so ETH(sdev) becomes invalid here.

What happens when a primary process closes a device before a secondary?
Is the secondary unable to stop / close its own then? Isn't there some
missing uninit?

This seems dangerous to me. Why not instead allocating a per-process
slab of memory that would hold the relevant references and outlive the
shared data (a per-process rte_eth_dev private data...).

>  		sdev->state = DEV_PROBED;
>  		/* fallthrough */
>  	case DEV_PROBED:
> @@ -287,7 +288,7 @@ fs_dev_remove(struct sub_device *sdev)
>  			ERROR("Bus detach failed for sub_device %u",
>  			      SUB_ID(sdev));
>  		} else {
> -			rte_eth_dev_release_port(ETH(sdev));
> +			rte_eth_dev_release_port(edev);
>  		}
>  		sdev->state = DEV_PARSED;
>  		/* fallthrough */
> diff --git a/drivers/net/failsafe/failsafe_private.h b/drivers/net/failsafe/failsafe_private.h
> index 84e847f..1e2ad2d 100644
> --- a/drivers/net/failsafe/failsafe_private.h
> +++ b/drivers/net/failsafe/failsafe_private.h
> @@ -100,13 +100,16 @@ struct fs_stats {
>  	uint64_t timestamp;
>  };
>  
> +/*
> + * Allocated in shared memory.
> + */
>  struct sub_device {
>  	/* Exhaustive DPDK device description */
>  	struct sub_device *next;
>  	struct rte_devargs devargs;
> -	struct rte_bus *bus;
> -	struct rte_device *dev;
> -	struct rte_eth_dev *edev;
> +	struct rte_bus *bus; /* per process. */
> +	struct rte_device *dev; /* per process. */
> +	struct rte_eth_dev_data *data; /* shared between processes */
>  	uint8_t sid;
>  	/* Device state machine */
>  	enum dev_state state;
> @@ -139,7 +142,7 @@ struct fs_priv {
>  	 * subs[0] is the preferred device
>  	 * any other is just another slave
>  	 */
> -	struct sub_device *subs;
> +	struct sub_device *subs;  /* shared between processes */
>  	uint8_t subs_head; /* if head == tail, no subs */
>  	uint8_t subs_tail; /* first invalid */
>  	uint8_t subs_tx; /* current emitting device */
> @@ -254,7 +257,7 @@ extern int failsafe_mac_from_arg;
>  
>  /* sdev: (struct sub_device *) */
>  #define ETH(sdev) \
> -	((sdev)->edev)
> +	((sdev)->data == NULL ? NULL : &rte_eth_devices[(sdev)->data->port_id])
>  
>  /* sdev: (struct sub_device *) */
>  #define PORT_ID(sdev) \
> -- 
> 2.7.4
> 

-- 
Gaëtan Rivet
6WIND