Re: [PATCH 05/10] vdpa/nfp: add the live migration logic

[Maxime Coquelin <maxime.coquelin@redhat.com>]

On 4/26/24 09:48, Chaoyong He wrote:
> From: Xinying Yu <xinying.yu@corigine.com>
> 
> Add the basic logic of software live migration.
> 
> Unset the ring notify area to stop the direct IO datapath if the
> device support, then we can setup the vring relay to help the
> live migration.
> 
> Signed-off-by: Xinying Yu <xinying.yu@corigine.com>
> Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
> Reviewed-by: Long Wu <long.wu@corigine.com>
> Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
> ---
>   drivers/vdpa/nfp/nfp_vdpa.c      | 66 +++++++++++++++++++++++++++++++-
>   drivers/vdpa/nfp/nfp_vdpa_core.c |  3 ++
>   drivers/vdpa/nfp/nfp_vdpa_core.h |  4 ++
>   3 files changed, 71 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/vdpa/nfp/nfp_vdpa.c b/drivers/vdpa/nfp/nfp_vdpa.c
> index cef80b5476..45092cb0af 100644
> --- a/drivers/vdpa/nfp/nfp_vdpa.c
> +++ b/drivers/vdpa/nfp/nfp_vdpa.c
> @@ -603,6 +603,30 @@ update_datapath(struct nfp_vdpa_dev *device)
>   	return ret;
>   }
>   
> +static int
> +nfp_vdpa_sw_fallback(struct nfp_vdpa_dev *device)
> +{
> +	int ret;
> +	int vid = device->vid;
> +
> +	/* Stop the direct IO data path */
> +	nfp_vdpa_unset_notify_relay(device);
> +	nfp_vdpa_disable_vfio_intr(device);
> +
> +	ret = rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, false);
> +	if ((ret != 0) && (ret != -ENOTSUP)) {
> +		DRV_VDPA_LOG(ERR, "Unset the host notifier failed.");
> +		goto error;
> +	}
> +
> +	device->hw.sw_fallback_running = true;
> +
> +	return 0;
> +
> +error:
> +	return ret;
> +}
> +
>   static int
>   nfp_vdpa_dev_config(int vid)
>   {
> @@ -646,8 +670,18 @@ nfp_vdpa_dev_close(int vid)
>   	}
>   
>   	device = node->device;
> -	rte_atomic_store_explicit(&device->dev_attached, 0, rte_memory_order_relaxed);
> -	update_datapath(device);
> +	if (device->hw.sw_fallback_running) {
> +		device->hw.sw_fallback_running = false;
> +
> +		rte_atomic_store_explicit(&device->dev_attached, 0,
> +				rte_memory_order_relaxed);
> +		rte_atomic_store_explicit(&device->running, 0,
> +				rte_memory_order_relaxed);
> +	} else {
> +		rte_atomic_store_explicit(&device->dev_attached, 0,
> +				rte_memory_order_relaxed);
> +		update_datapath(device);
> +	}
>   
>   	return 0;
>   }
> @@ -770,7 +804,35 @@ nfp_vdpa_get_protocol_features(struct rte_vdpa_device *vdev __rte_unused,
>   static int
>   nfp_vdpa_set_features(int32_t vid)
>   {
> +	int ret;
> +	uint64_t features = 0;
> +	struct nfp_vdpa_dev *device;
> +	struct rte_vdpa_device *vdev;
> +	struct nfp_vdpa_dev_node *node;
> +
>   	DRV_VDPA_LOG(DEBUG, "Start vid=%d", vid);
> +
> +	vdev = rte_vhost_get_vdpa_device(vid);
> +	node = nfp_vdpa_find_node_by_vdev(vdev);
> +	if (node == NULL) {
> +		DRV_VDPA_LOG(ERR, "Invalid vDPA device: %p", vdev);
> +		return -ENODEV;
> +	}
> +
> +	rte_vhost_get_negotiated_features(vid, &features);
> +
> +	if (RTE_VHOST_NEED_LOG(features) == 0)
> +		return 0;
> +
> +	device = node->device;
> +	if (device->hw.sw_lm) {
> +		ret = nfp_vdpa_sw_fallback(device);
> +		if (ret != 0) {
> +			DRV_VDPA_LOG(ERR, "Software fallback start failed");
> +			return -1;
> +		}
> +	}
> +
>   	return 0;
>   }
>   
> diff --git a/drivers/vdpa/nfp/nfp_vdpa_core.c b/drivers/vdpa/nfp/nfp_vdpa_core.c
> index 79ecd2b4fc..82a323a6d0 100644
> --- a/drivers/vdpa/nfp/nfp_vdpa_core.c
> +++ b/drivers/vdpa/nfp/nfp_vdpa_core.c
> @@ -91,8 +91,11 @@ nfp_vdpa_hw_init(struct nfp_vdpa_hw *vdpa_hw,
>   	tx_bar = (uint8_t *)pci_dev->mem_resource[2].addr + tx_bar_off;
>   	hw->qcp_cfg = tx_bar + NFP_QCP_QUEUE_ADDR_SZ;
>   
> +	vdpa_hw->sw_lm = true;
> +
>   	vdpa_hw->features = (1ULL << VIRTIO_F_VERSION_1) |
>   			(1ULL << VIRTIO_F_IN_ORDER) |
> +			(1ULL << VHOST_F_LOG_ALL) |
>   			(1ULL << VHOST_USER_F_PROTOCOL_FEATURES);

Is the live-migration works if we stop at patch 5?
If not, I would suggest to enable the feature only at the end of the
series.

>   
>   	return 0;
> diff --git a/drivers/vdpa/nfp/nfp_vdpa_core.h b/drivers/vdpa/nfp/nfp_vdpa_core.h
> index a8e0d6dd70..0f880fc0c6 100644
> --- a/drivers/vdpa/nfp/nfp_vdpa_core.h
> +++ b/drivers/vdpa/nfp/nfp_vdpa_core.h
> @@ -36,6 +36,10 @@ struct nfp_vdpa_hw {
>   	uint8_t mac_addr[RTE_ETHER_ADDR_LEN];
>   	uint8_t notify_region;
>   	uint8_t nr_vring;
> +
> +	/** Software Live Migration */
> +	bool sw_lm;
> +	bool sw_fallback_running;
>   };
>   
>   int nfp_vdpa_hw_init(struct nfp_vdpa_hw *vdpa_hw, struct rte_pci_device *dev);