From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by dpdk.org (Postfix) with ESMTP id 4F737728A for ; Mon, 19 Mar 2018 11:12:59 +0100 (CET) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by fmsmga103.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 19 Mar 2018 03:12:58 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.48,330,1517904000"; d="scan'208";a="38658289" Received: from unknown (HELO dpdk99.sh.intel.com) ([10.67.110.156]) by fmsmga004.fm.intel.com with ESMTP; 19 Mar 2018 03:12:57 -0700 From: Zhihong Wang To: dev@dpdk.org Cc: jianfeng.tan@intel.com, tiwei.bie@intel.com, maxime.coquelin@redhat.com, yliu@fridaylinux.org, cunming.liang@intel.com, xiao.w.wang@intel.com, dan.daly@intel.com, Zhihong Wang Message-Id: <20180227101342.18521-5-zhihong.wang@intel.com> X-Mailer: git-send-email 2.13.6 In-Reply-To: <20180227101342.18521-1-zhihong.wang@intel.com> References: <1517614137-62926-1-git-send-email-zhihong.wang@intel.com> <20180227101342.18521-1-zhihong.wang@intel.com> Subject: [dpdk-dev] [PATCH v3 4/5] vhost: adapt vhost lib for selective datapath X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Date: Mon, 19 Mar 2018 10:12:59 -0000 X-Original-Date: Tue, 27 Feb 2018 18:13:41 +0800 X-List-Received-Date: Mon, 19 Mar 2018 10:12:59 -0000 This patch adapts vhost lib for selective datapath by calling device ops at the corresponding stage. Signed-off-by: Zhihong Wang --- Changes in v2: 1. Ensure negotiated capabilities are supported in vhost-user lib. 2. Configure the data path at the right time. lib/librte_vhost/rte_vhost.h | 27 ++++++++++++ lib/librte_vhost/rte_vhost_version.map | 2 + lib/librte_vhost/socket.c | 76 +++++++++++++++++++++++++++++++++- lib/librte_vhost/vhost.c | 3 ++ lib/librte_vhost/vhost.h | 2 + lib/librte_vhost/vhost_user.c | 56 ++++++++++++++++++++++--- 6 files changed, 158 insertions(+), 8 deletions(-) diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h index 7aa57ca87..77c2a1a8b 100644 --- a/lib/librte_vhost/rte_vhost.h +++ b/lib/librte_vhost/rte_vhost.h @@ -303,6 +303,33 @@ int rte_vhost_driver_disable_features(const char *path, uint64_t features); int rte_vhost_driver_get_features(const char *path, uint64_t *features); /** + * Get the protocol feature bits before feature negotiation. + * + * @param path + * The vhost-user socket file path + * @param protocol_features + * A pointer to store the queried protocol feature bits + * @return + * 0 on success, -1 on failure + */ +int __rte_experimental +rte_vhost_driver_get_protocol_features(const char *path, + uint64_t *protocol_features); + +/** + * Get the queue number bits before feature negotiation. + * + * @param path + * The vhost-user socket file path + * @param queue_num + * A pointer to store the queried queue number bits + * @return + * 0 on success, -1 on failure + */ +int __rte_experimental +rte_vhost_driver_get_queue_num(const char *path, uint32_t *queue_num); + +/** * Get the feature bits after negotiation * * @param vid diff --git a/lib/librte_vhost/rte_vhost_version.map b/lib/librte_vhost/rte_vhost_version.map index c505596c5..8ef2a396c 100644 --- a/lib/librte_vhost/rte_vhost_version.map +++ b/lib/librte_vhost/rte_vhost_version.map @@ -72,4 +72,6 @@ EXPERIMENTAL { rte_vhost_driver_get_vdpa_did; rte_vhost_get_vdpa_eid; rte_vhost_get_vdpa_did; + rte_vhost_driver_get_protocol_features; + rte_vhost_driver_get_queue_num; } DPDK_18.02; diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c index 8551eb58c..14dce2a73 100644 --- a/lib/librte_vhost/socket.c +++ b/lib/librte_vhost/socket.c @@ -216,6 +216,9 @@ vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket) vhost_set_builtin_virtio_net(vid, vsocket->use_builtin_virtio_net); + vhost_set_vdpa_eid(vid, vsocket->eid); + vhost_set_vdpa_did(vid, vsocket->did); + if (vsocket->dequeue_zero_copy) vhost_enable_dequeue_zero_copy(vid); @@ -677,11 +680,80 @@ int rte_vhost_driver_get_features(const char *path, uint64_t *features) { struct vhost_user_socket *vsocket; + struct rte_vdpa_eng_attr attr; + int eid = -1; pthread_mutex_lock(&vhost_user.mutex); vsocket = find_vhost_user_socket(path); - if (vsocket) - *features = vsocket->features; + if (vsocket) { + eid = vsocket->eid; + if (rte_vdpa_info_query(eid, &attr) < 0) + *features = vsocket->features; + else + *features = vsocket->features & attr.features; + + } + pthread_mutex_unlock(&vhost_user.mutex); + + if (!vsocket) { + RTE_LOG(ERR, VHOST_CONFIG, + "socket file %s is not registered yet.\n", path); + return -1; + } else { + return 0; + } +} + +int +rte_vhost_driver_get_protocol_features(const char *path, + uint64_t *protocol_features) +{ + struct vhost_user_socket *vsocket; + struct rte_vdpa_eng_attr attr; + int eid = -1; + + pthread_mutex_lock(&vhost_user.mutex); + vsocket = find_vhost_user_socket(path); + if (vsocket) { + eid = vsocket->eid; + if (rte_vdpa_info_query(eid, &attr) < 0) + *protocol_features = VHOST_USER_PROTOCOL_FEATURES; + else + *protocol_features = VHOST_USER_PROTOCOL_FEATURES + & attr.protocol_features; + + } + pthread_mutex_unlock(&vhost_user.mutex); + + if (!vsocket) { + RTE_LOG(ERR, VHOST_CONFIG, + "socket file %s is not registered yet.\n", path); + return -1; + } else { + return 0; + } +} + +int +rte_vhost_driver_get_queue_num(const char *path, + uint32_t *queue_num) +{ + struct vhost_user_socket *vsocket; + struct rte_vdpa_eng_attr attr; + int eid = -1; + + pthread_mutex_lock(&vhost_user.mutex); + vsocket = find_vhost_user_socket(path); + if (vsocket) { + eid = vsocket->eid; + if (rte_vdpa_info_query(eid, &attr) < 0) + *queue_num = VHOST_MAX_QUEUE_PAIRS; + else if (attr.queue_num > VHOST_MAX_QUEUE_PAIRS) + *queue_num = VHOST_MAX_QUEUE_PAIRS; + else + *queue_num = attr.queue_num; + + } pthread_mutex_unlock(&vhost_user.mutex); if (!vsocket) { diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c index 45cf90f99..f8a5a1c42 100644 --- a/lib/librte_vhost/vhost.c +++ b/lib/librte_vhost/vhost.c @@ -297,11 +297,14 @@ void vhost_destroy_device(int vid) { struct virtio_net *dev = get_device(vid); + int eid = dev->eid; if (dev == NULL) return; if (dev->flags & VIRTIO_DEV_RUNNING) { + if (eid >= 0 && vdpa_engines[eid]->eng_drv->dev_ops.dev_close) + vdpa_engines[eid]->eng_drv->dev_ops.dev_close(dev->vid); dev->flags &= ~VIRTIO_DEV_RUNNING; dev->notify_ops->destroy_device(vid); } diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index e06d789fa..53fd76e92 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -27,6 +27,8 @@ #define VIRTIO_DEV_READY 2 /* Used to indicate that the built-in vhost net device backend is enabled */ #define VIRTIO_DEV_BUILTIN_VIRTIO_NET 4 +/* Used to indicate that the device has its own data path and configured */ +#define VIRTIO_DEV_VDPA_CONFIGURED 8 /* Backend value set by guest. */ #define VIRTIO_DEV_STOPPED -1 diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c index 5c5361066..d4e0c9c62 100644 --- a/lib/librte_vhost/vhost_user.c +++ b/lib/librte_vhost/vhost_user.c @@ -133,7 +133,11 @@ vhost_user_set_owner(void) static int vhost_user_reset_owner(struct virtio_net *dev) { + int eid = dev->eid; + if (dev->flags & VIRTIO_DEV_RUNNING) { + if (eid >= 0 && vdpa_engines[eid]->eng_drv->dev_ops.dev_close) + vdpa_engines[eid]->eng_drv->dev_ops.dev_close(dev->vid); dev->flags &= ~VIRTIO_DEV_RUNNING; dev->notify_ops->destroy_device(dev->vid); } @@ -156,12 +160,25 @@ vhost_user_get_features(struct virtio_net *dev) } /* + * The queue number that we support are requested. + */ +static uint32_t +vhost_user_get_queue_num(struct virtio_net *dev) +{ + uint32_t queue_num = 0; + + rte_vhost_driver_get_queue_num(dev->ifname, &queue_num); + return (uint64_t)queue_num; +} + +/* * We receive the negotiated features supported by us and the virtio device. */ static int vhost_user_set_features(struct virtio_net *dev, uint64_t features) { uint64_t vhost_features = 0; + int eid = dev->eid; rte_vhost_driver_get_features(dev->ifname, &vhost_features); if (features & ~vhost_features) { @@ -191,6 +208,9 @@ vhost_user_set_features(struct virtio_net *dev, uint64_t features) dev->notify_ops->features_changed(dev->vid, features); } + if (eid >= 0 && vdpa_engines[eid]->eng_drv->dev_ops.feature_set) + vdpa_engines[eid]->eng_drv->dev_ops.feature_set(dev->vid); + dev->features = features; if (dev->features & ((1 << VIRTIO_NET_F_MRG_RXBUF) | (1ULL << VIRTIO_F_VERSION_1))) { @@ -933,9 +953,12 @@ vhost_user_get_vring_base(struct virtio_net *dev, VhostUserMsg *msg) { struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index]; + int eid = dev->eid; /* We have to stop the queue (virtio) if it is running. */ if (dev->flags & VIRTIO_DEV_RUNNING) { + if (eid >= 0 && vdpa_engines[eid]->eng_drv->dev_ops.dev_close) + vdpa_engines[eid]->eng_drv->dev_ops.dev_close(dev->vid); dev->flags &= ~VIRTIO_DEV_RUNNING; dev->notify_ops->destroy_device(dev->vid); } @@ -983,16 +1006,22 @@ vhost_user_set_vring_enable(struct virtio_net *dev, VhostUserMsg *msg) { int enable = (int)msg->payload.state.num; + int index = (int)msg->payload.state.index; + int eid = dev->eid; RTE_LOG(INFO, VHOST_CONFIG, "set queue enable: %d to qp idx: %d\n", - enable, msg->payload.state.index); + enable, index); + + if (eid >= 0 && vdpa_engines[eid]->eng_drv->dev_ops.vring_state_set) + vdpa_engines[eid]->eng_drv->dev_ops.vring_state_set(dev->vid, + index, enable); if (dev->notify_ops->vring_state_changed) dev->notify_ops->vring_state_changed(dev->vid, - msg->payload.state.index, enable); + index, enable); - dev->virtqueue[msg->payload.state.index]->enabled = enable; + dev->virtqueue[index]->enabled = enable; return 0; } @@ -1001,9 +1030,10 @@ static void vhost_user_get_protocol_features(struct virtio_net *dev, struct VhostUserMsg *msg) { - uint64_t features, protocol_features = VHOST_USER_PROTOCOL_FEATURES; + uint64_t features, protocol_features; rte_vhost_driver_get_features(dev->ifname, &features); + rte_vhost_driver_get_protocol_features(dev->ifname, &protocol_features); /* * REPLY_ACK protocol feature is only mandatory for now @@ -1014,7 +1044,6 @@ vhost_user_get_protocol_features(struct virtio_net *dev, if (!(features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK); - msg->payload.u64 = protocol_features; msg->size = sizeof(msg->payload.u64); } @@ -1099,6 +1128,7 @@ static int vhost_user_send_rarp(struct virtio_net *dev, struct VhostUserMsg *msg) { uint8_t *mac = (uint8_t *)&msg->payload.u64; + int eid = dev->eid; RTE_LOG(DEBUG, VHOST_CONFIG, ":: mac: %02x:%02x:%02x:%02x:%02x:%02x\n", @@ -1114,6 +1144,8 @@ vhost_user_send_rarp(struct virtio_net *dev, struct VhostUserMsg *msg) */ rte_smp_wmb(); rte_atomic16_set(&dev->broadcast_rarp, 1); + if (eid >= 0 && vdpa_engines[eid]->eng_drv->dev_ops.migration_done) + vdpa_engines[eid]->eng_drv->dev_ops.migration_done(dev->vid); return 0; } @@ -1375,6 +1407,8 @@ vhost_user_msg_handler(int vid, int fd) { struct virtio_net *dev; struct VhostUserMsg msg; + struct rte_vdpa_engine *eng; + int eid; int ret; int unlock_required = 0; @@ -1527,7 +1561,7 @@ vhost_user_msg_handler(int vid, int fd) break; case VHOST_USER_GET_QUEUE_NUM: - msg.payload.u64 = VHOST_MAX_QUEUE_PAIRS; + msg.payload.u64 = (uint64_t)vhost_user_get_queue_num(dev); msg.size = sizeof(msg.payload.u64); send_vhost_reply(fd, &msg); break; @@ -1580,6 +1614,16 @@ vhost_user_msg_handler(int vid, int fd) } } + eid = dev->eid; + if (eid >= 0 && virtio_is_ready(dev) && + !(dev->flags & VIRTIO_DEV_VDPA_CONFIGURED) && + msg.request.master == VHOST_USER_SET_VRING_ENABLE) { + eng = vdpa_engines[eid]; + if (eng->eng_drv->dev_ops.dev_conf) + eng->eng_drv->dev_ops.dev_conf(vid); + dev->flags |= VIRTIO_DEV_VDPA_CONFIGURED; + } + return 0; } -- 2.13.6