From: Chenbo Xia <chenbo.xia@intel.com> To: dev@dpdk.org, thomas@monjalon.net, david.marchand@redhat.com Cc: stephen@networkplumber.org, cunming.liang@intel.com, xiuchun.lu@intel.com, miao.li@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Subject: [dpdk-dev] [PATCH v2 3/9] vfio_user: implement device and region related APIs Date: Thu, 14 Jan 2021 14:14:05 +0800 Message-ID: <20210114061411.39166-4-chenbo.xia@intel.com> (raw) In-Reply-To: <20210114061411.39166-1-chenbo.xia@intel.com> This patch introduces device and region related APIs, which are rte_vfio_user_set_dev_info() and rte_vfio_user_set_reg_info(). The corresponding vfio-user command handling is also added with the definition of all vfio-user command identity. Signed-off-by: Chenbo Xia <chenbo.xia@intel.com> Signed-off-by: Xiuchun Lu <xiuchun.lu@intel.com> --- lib/librte_vfio_user/rte_vfio_user.h | 62 +++++++ lib/librte_vfio_user/version.map | 2 + lib/librte_vfio_user/vfio_user_base.c | 12 ++ lib/librte_vfio_user/vfio_user_base.h | 32 +++- lib/librte_vfio_user/vfio_user_server.c | 235 ++++++++++++++++++++++++ lib/librte_vfio_user/vfio_user_server.h | 2 + 6 files changed, 344 insertions(+), 1 deletion(-) diff --git a/lib/librte_vfio_user/rte_vfio_user.h b/lib/librte_vfio_user/rte_vfio_user.h index 705a2f6632..117e994cc6 100644 --- a/lib/librte_vfio_user/rte_vfio_user.h +++ b/lib/librte_vfio_user/rte_vfio_user.h @@ -5,13 +5,35 @@ #ifndef _RTE_VFIO_USER_H #define _RTE_VFIO_USER_H +#include <linux/vfio.h> + #include <rte_compat.h> +struct rte_vfio_user_reg_info; + +typedef ssize_t (*rte_vfio_user_reg_acc_t)(struct rte_vfio_user_reg_info *reg, + char *buf, size_t count, loff_t pos, bool iswrite); + +struct rte_vfio_user_reg_info { + rte_vfio_user_reg_acc_t rw; + void *base; + int fd; + struct vfio_region_info *info; + void *priv; +}; + +struct rte_vfio_user_regions { + uint32_t reg_num; + struct rte_vfio_user_reg_info reg_info[]; +}; + /** * Below APIs are for vfio-user server (device provider) to use: * *rte_vfio_user_register * *rte_vfio_user_unregister * *rte_vfio_user_start + * *rte_vfio_user_set_dev_info + * *rte_vfio_user_set_reg_info */ /** @@ -51,4 +73,44 @@ __rte_experimental int rte_vfio_user_start(const char *sock_addr); +/** + * Set the device information for a vfio-user device. + * + * This information must be set before calling rte_vfio_user_start, and should + * not be updated after start. Update after start can be done by unregistration + * and re-registration, and then the device-level change can be detected by + * vfio-user client. + * + * @param sock_addr + * Unix domain socket address + * @param dev_info + * Device information for the vfio-user device + * @return + * 0 on success, -1 on failure + */ +__rte_experimental +int +rte_vfio_user_set_dev_info(const char *sock_addr, + struct vfio_device_info *dev_info); + +/** + * Set the region information for a vfio-user device. + * + * This information must be set before calling rte_vfio_user_start, and should + * not be updated after start. Update after start can be done by unregistration + * and re-registration, and then the device-level change can be detected by + * vfio-user client. + * + * @param sock_addr + * Unix domain socket address + * @param reg + * Region information for the vfio-user device + * @return + * 0 on success, -1 on failure + */ +__rte_experimental +int +rte_vfio_user_set_reg_info(const char *sock_addr, + struct rte_vfio_user_regions *reg); + #endif diff --git a/lib/librte_vfio_user/version.map b/lib/librte_vfio_user/version.map index e53095eda8..0f4f5acba5 100644 --- a/lib/librte_vfio_user/version.map +++ b/lib/librte_vfio_user/version.map @@ -4,6 +4,8 @@ EXPERIMENTAL { rte_vfio_user_register; rte_vfio_user_unregister; rte_vfio_user_start; + rte_vfio_user_set_dev_info; + rte_vfio_user_set_reg_info; local: *; }; diff --git a/lib/librte_vfio_user/vfio_user_base.c b/lib/librte_vfio_user/vfio_user_base.c index b9fdff5b02..7badca23b7 100644 --- a/lib/librte_vfio_user/vfio_user_base.c +++ b/lib/librte_vfio_user/vfio_user_base.c @@ -13,6 +13,18 @@ int vfio_user_log_level; const char *vfio_user_msg_str[VFIO_USER_MAX] = { [VFIO_USER_NONE] = "VFIO_USER_NONE", [VFIO_USER_VERSION] = "VFIO_USER_VERSION", + [VFIO_USER_DMA_MAP] = "VFIO_USER_DMA_MAP", + [VFIO_USER_DMA_UNMAP] = "VFIO_USER_DMA_UNMAP", + [VFIO_USER_DEVICE_GET_INFO] = "VFIO_USER_DEVICE_GET_INFO", + [VFIO_USER_DEVICE_GET_REGION_INFO] = "VFIO_USER_GET_REGION_INFO", + [VFIO_USER_DEVICE_GET_IRQ_INFO] = "VFIO_USER_DEVICE_GET_IRQ_INFO", + [VFIO_USER_DEVICE_SET_IRQS] = "VFIO_USER_DEVICE_SET_IRQS", + [VFIO_USER_REGION_READ] = "VFIO_USER_REGION_READ", + [VFIO_USER_REGION_WRITE] = "VFIO_USER_REGION_WRITE", + [VFIO_USER_DMA_READ] = "VFIO_USER_DMA_READ", + [VFIO_USER_DMA_WRITE] = "VFIO_USER_DMA_WRITE", + [VFIO_USER_VM_INTERRUPT] = "VFIO_USER_VM_INTERRUPT", + [VFIO_USER_DEVICE_RESET] = "VFIO_USER_DEVICE_RESET", }; void diff --git a/lib/librte_vfio_user/vfio_user_base.h b/lib/librte_vfio_user/vfio_user_base.h index f9b0b94665..f92886b56a 100644 --- a/lib/librte_vfio_user/vfio_user_base.h +++ b/lib/librte_vfio_user/vfio_user_base.h @@ -11,6 +11,8 @@ #define VFIO_USER_VERSION_MAJOR 1 #define VFIO_USER_VERSION_MINOR 0 +#define VFIO_USER_MAX_RSVD 512 +#define VFIO_USER_MAX_RW_DATA 512 #define VFIO_USER_MAX_FD 1024 #define VFIO_USER_MAX_VERSION_DATA 512 @@ -30,7 +32,19 @@ struct vfio_user_socket { typedef enum VFIO_USER_CMD_TYPE { VFIO_USER_NONE = 0, VFIO_USER_VERSION = 1, - VFIO_USER_MAX = 2, + VFIO_USER_DMA_MAP = 2, + VFIO_USER_DMA_UNMAP = 3, + VFIO_USER_DEVICE_GET_INFO = 4, + VFIO_USER_DEVICE_GET_REGION_INFO = 5, + VFIO_USER_DEVICE_GET_IRQ_INFO = 6, + VFIO_USER_DEVICE_SET_IRQS = 7, + VFIO_USER_REGION_READ = 8, + VFIO_USER_REGION_WRITE = 9, + VFIO_USER_DMA_READ = 10, + VFIO_USER_DMA_WRITE = 11, + VFIO_USER_VM_INTERRUPT = 12, + VFIO_USER_DEVICE_RESET = 13, + VFIO_USER_MAX = 14, } VFIO_USER_CMD_TYPE; struct vfio_user_version { @@ -40,6 +54,19 @@ struct vfio_user_version { uint8_t ver_data[VFIO_USER_MAX_VERSION_DATA]; }; +struct vfio_user_reg { + struct vfio_region_info reg_info; + /* Reserved for region capability list */ + uint8_t rsvd[VFIO_USER_MAX_RSVD]; +}; + +struct vfio_user_reg_rw { + uint64_t reg_offset; + uint32_t reg_idx; + uint32_t size; + char data[VFIO_USER_MAX_RW_DATA]; +}; + struct vfio_user_msg { uint16_t msg_id; uint16_t cmd; @@ -52,6 +79,9 @@ struct vfio_user_msg { uint32_t err; /* Valid in reply, optional */ union { struct vfio_user_version ver; + struct vfio_device_info dev_info; + struct vfio_user_reg reg_info; + struct vfio_user_reg_rw reg_rw; } payload; int fds[VFIO_USER_MAX_FD]; int fd_num; diff --git a/lib/librte_vfio_user/vfio_user_server.c b/lib/librte_vfio_user/vfio_user_server.c index 35544c819a..aab923e727 100644 --- a/lib/librte_vfio_user/vfio_user_server.c +++ b/lib/librte_vfio_user/vfio_user_server.c @@ -5,6 +5,7 @@ #include <unistd.h> #include <fcntl.h> #include <pthread.h> +#include <inttypes.h> #include <sys/socket.h> #include <sys/un.h> @@ -39,9 +40,159 @@ vfio_user_negotiate_version(struct vfio_user_server *dev, return -ENOTSUP; } +static int +vfio_user_device_get_info(struct vfio_user_server *dev, + struct vfio_user_msg *msg) +{ + struct vfio_device_info *dev_info = &msg->payload.dev_info; + + if (vfio_user_check_msg_fdnum(msg, 0) != 0) + return -EINVAL; + + if (msg->size != sizeof(*dev_info) + VFIO_USER_MSG_HDR_SIZE) { + VFIO_USER_LOG(ERR, "Invalid message for get dev info\n"); + return -EINVAL; + } + + memcpy(dev_info, dev->dev_info, sizeof(*dev_info)); + + VFIO_USER_LOG(DEBUG, "Device info: argsz(0x%x), flags(0x%x), " + "regions(%u), irqs(%u)\n", dev_info->argsz, dev_info->flags, + dev_info->num_regions, dev_info->num_irqs); + + return 0; +} + +static int +vfio_user_device_get_reg_info(struct vfio_user_server *dev, + struct vfio_user_msg *msg) +{ + struct vfio_user_reg *reg = &msg->payload.reg_info; + struct rte_vfio_user_reg_info *reg_info; + struct vfio_region_info *vinfo; + + if (vfio_user_check_msg_fdnum(msg, 0) != 0) + return -EINVAL; + + if (msg->size > sizeof(*reg) + VFIO_USER_MSG_HDR_SIZE || + dev->reg->reg_num <= reg->reg_info.index) { + VFIO_USER_LOG(ERR, "Invalid message for get region info\n"); + return -EINVAL; + } + + reg_info = &dev->reg->reg_info[reg->reg_info.index]; + vinfo = reg_info->info; + memcpy(reg, vinfo, vinfo->argsz); + + if (reg_info->fd != -1) { + msg->fd_num = 1; + msg->fds[0] = reg_info->fd; + } + + VFIO_USER_LOG(DEBUG, "Region(%u) info: addr(0x%" PRIx64 "), fd(%d), " + "sz(0x%llx), argsz(0x%x), c_off(0x%x), flags(0x%x) " + "off(0x%llx)\n", vinfo->index, (uint64_t)reg_info->base, + reg_info->fd, vinfo->size, vinfo->argsz, vinfo->cap_offset, + vinfo->flags, vinfo->offset); + + return 0; +} + +static int +vfio_user_region_read(struct vfio_user_server *dev, + struct vfio_user_msg *msg) +{ + struct vfio_user_reg_rw *rw = &msg->payload.reg_rw; + struct rte_vfio_user_regions *reg = dev->reg; + struct rte_vfio_user_reg_info *reg_info; + size_t count; + + if (vfio_user_check_msg_fdnum(msg, 0) != 0) + return -EINVAL; + + reg_info = ®->reg_info[rw->reg_idx]; + + if (rw->reg_idx >= reg->reg_num || + rw->size > VFIO_USER_MAX_RW_DATA || + rw->reg_offset >= reg_info->info->size || + rw->reg_offset + rw->size > reg_info->info->size) { + VFIO_USER_LOG(ERR, "Invalid read region request\n"); + rw->size = 0; + return 0; + } + + VFIO_USER_LOG(DEBUG, "Read Region(%u): offset(0x%" PRIx64 ")," + "size(0x%x)\n", rw->reg_idx, rw->reg_offset, rw->size); + + if (reg_info->rw) { + count = reg_info->rw(reg_info, msg->payload.reg_rw.data, + rw->size, rw->reg_offset, 0); + rw->size = count; + msg->size += count; + return 0; + } + + memcpy(&msg->payload.reg_rw.data, + (uint8_t *)reg_info->base + rw->reg_offset, rw->size); + msg->size += rw->size; + return 0; +} + +static int +vfio_user_region_write(struct vfio_user_server *dev, + struct vfio_user_msg *msg) +{ + struct vfio_user_reg_rw *rw = &msg->payload.reg_rw; + struct rte_vfio_user_regions *reg = dev->reg; + struct rte_vfio_user_reg_info *reg_info; + size_t count; + + if (vfio_user_check_msg_fdnum(msg, 0) != 0) + return -EINVAL; + + if (rw->reg_idx >= reg->reg_num) { + VFIO_USER_LOG(ERR, "Write a non-existed region\n"); + return -EINVAL; + } + + reg_info = ®->reg_info[rw->reg_idx]; + + VFIO_USER_LOG(DEBUG, "Write Region(%u): offset(0x%" PRIx64 ")," + "size(0x%x)\n", rw->reg_idx, rw->reg_offset, rw->size); + + if (reg_info->rw) { + count = reg_info->rw(reg_info, msg->payload.reg_rw.data, + rw->size, rw->reg_offset, 1); + if (count < rw->size) { + VFIO_USER_LOG(ERR, "Write region %d failed\n", + rw->reg_idx); + return -EIO; + } + rw->size = 0; + return 0; + } + + memcpy((uint8_t *)reg_info->base + rw->reg_offset, + &msg->payload.reg_rw.data, rw->size); + rw->size = 0; + return 0; +} + static vfio_user_msg_handler_t vfio_user_msg_handlers[VFIO_USER_MAX] = { [VFIO_USER_NONE] = NULL, [VFIO_USER_VERSION] = vfio_user_negotiate_version, + [VFIO_USER_DMA_MAP] = NULL, + [VFIO_USER_DMA_UNMAP] = NULL, + [VFIO_USER_DEVICE_GET_INFO] = vfio_user_device_get_info, + [VFIO_USER_DEVICE_GET_REGION_INFO] = vfio_user_device_get_reg_info, + [VFIO_USER_DEVICE_GET_IRQ_INFO] = NULL, + [VFIO_USER_DEVICE_SET_IRQS] = NULL, + [VFIO_USER_REGION_READ] = vfio_user_region_read, + [VFIO_USER_REGION_WRITE] = vfio_user_region_write, + [VFIO_USER_DMA_READ] = NULL, + [VFIO_USER_DMA_WRITE] = NULL, + [VFIO_USER_VM_INTERRUPT] = NULL, + [VFIO_USER_DEVICE_RESET] = NULL, }; static struct vfio_user_server_socket * @@ -563,6 +714,13 @@ vfio_user_start_server(struct vfio_user_server_socket *sk) return 0; } + /* All the info must be set before start */ + if (!dev->dev_info || !dev->reg) { + VFIO_USER_LOG(ERR, "Failed to start, " + "dev/reg info must be set before start\n"); + return -1; + } + unlink(path); ret = bind(fd, (struct sockaddr *)&sk->un, sizeof(sk->un)); if (ret < 0) { @@ -705,3 +863,80 @@ rte_vfio_user_start(const char *sock_addr) pthread_mutex_unlock(&vfio_ep_sock.mutex); return -1; } + +static struct vfio_user_server * +vfio_user_find_stopped_server(const char *sock_addr) +{ + struct vfio_user_server *dev; + struct vfio_user_server_socket *sk; + int dev_id; + + pthread_mutex_lock(&vfio_ep_sock.mutex); + sk = vfio_user_find_socket(sock_addr); + pthread_mutex_unlock(&vfio_ep_sock.mutex); + + if (!sk) { + VFIO_USER_LOG(ERR, "Failed to find server with sock_addr " + "%s: addr not registered.\n", sock_addr); + return NULL; + } + + dev_id = sk->sock.dev_id; + dev = vfio_user_get_device(dev_id); + if (!dev) { + VFIO_USER_LOG(ERR, "Failed to find server: " + "device %d not found.\n", dev_id); + return NULL; + } + + if (dev->started) { + VFIO_USER_LOG(ERR, "Failed to find stopped server: " + "device %d already started\n", dev_id); + return NULL; + } + + return dev; +} + +int +rte_vfio_user_set_dev_info(const char *sock_addr, + struct vfio_device_info *dev_info) +{ + struct vfio_user_server *dev; + + if (!dev_info) + return -1; + + dev = vfio_user_find_stopped_server(sock_addr); + if (!dev) { + VFIO_USER_LOG(ERR, "Failed to set device(%s) information: " + "cannot find stopped server\n", sock_addr); + return -1; + } + + dev->dev_info = dev_info; + + return 0; +} + +int +rte_vfio_user_set_reg_info(const char *sock_addr, + struct rte_vfio_user_regions *reg) +{ + struct vfio_user_server *dev; + + if (!reg) + return -1; + + dev = vfio_user_find_stopped_server(sock_addr); + if (!dev) { + VFIO_USER_LOG(ERR, "Failed to set region information for " + "device with sock(%s): cannot find stopped server\n", + sock_addr); + return -1; + } + + dev->reg = reg; + + return 0; +} diff --git a/lib/librte_vfio_user/vfio_user_server.h b/lib/librte_vfio_user/vfio_user_server.h index 0a5b17584a..4e7337113c 100644 --- a/lib/librte_vfio_user/vfio_user_server.h +++ b/lib/librte_vfio_user/vfio_user_server.h @@ -16,6 +16,8 @@ struct vfio_user_server { uint32_t msg_id; char sock_addr[PATH_MAX]; struct vfio_user_version ver; + struct vfio_device_info *dev_info; + struct rte_vfio_user_regions *reg; }; typedef int (*event_handler)(int fd, void *data); -- 2.17.1
next prev parent reply other threads:[~2021-01-14 6:19 UTC|newest] Thread overview: 43+ messages / expand[flat|nested] mbox.gz Atom feed top 2020-12-18 7:38 [dpdk-dev] [PATCH 0/9] Introduce vfio-user library Chenbo Xia 2020-12-18 7:38 ` [dpdk-dev] [PATCH 1/9] lib: introduce " Chenbo Xia 2020-12-18 17:13 ` Stephen Hemminger 2020-12-19 6:12 ` Xia, Chenbo 2020-12-18 17:17 ` Stephen Hemminger 2020-12-19 6:25 ` Xia, Chenbo 2020-12-18 7:38 ` [dpdk-dev] [PATCH 2/9] vfio_user: implement lifecycle related APIs Chenbo Xia 2021-01-05 8:34 ` Xing, Beilei 2021-01-05 9:58 ` Xia, Chenbo 2020-12-18 7:38 ` [dpdk-dev] [PATCH 3/9] vfio_user: implement device and region " Chenbo Xia 2021-01-06 5:51 ` Xing, Beilei 2021-01-06 7:50 ` Xia, Chenbo 2020-12-18 7:38 ` [dpdk-dev] [PATCH 4/9] vfio_user: implement DMA table and socket address API Chenbo Xia 2020-12-18 7:38 ` [dpdk-dev] [PATCH 5/9] vfio_user: implement interrupt related APIs Chenbo Xia 2020-12-30 1:04 ` Wu, Jingjing 2020-12-30 2:31 ` Xia, Chenbo 2020-12-18 7:38 ` [dpdk-dev] [PATCH 6/9] vfio_user: add client APIs of device attach/detach Chenbo Xia 2020-12-18 7:38 ` [dpdk-dev] [PATCH 7/9] vfio_user: add client APIs of DMA/IRQ/region Chenbo Xia 2021-01-07 2:41 ` Xing, Beilei 2021-01-07 7:26 ` Xia, Chenbo 2020-12-18 7:38 ` [dpdk-dev] [PATCH 8/9] test/vfio_user: introduce functional test Chenbo Xia 2020-12-18 7:38 ` [dpdk-dev] [PATCH 9/9] doc: add vfio-user library guide Chenbo Xia 2021-01-06 5:07 ` Xing, Beilei 2021-01-06 7:43 ` Xia, Chenbo 2020-12-18 9:37 ` [dpdk-dev] [PATCH 0/9] Introduce vfio-user library David Marchand 2020-12-18 14:07 ` Thanos Makatos 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 " Chenbo Xia 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 1/9] lib: introduce " Chenbo Xia 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 2/9] vfio_user: implement lifecycle related APIs Chenbo Xia 2021-01-14 6:14 ` Chenbo Xia [this message] 2021-01-14 18:48 ` [dpdk-dev] [PATCH v2 3/9] vfio_user: implement device and region " David Christensen 2021-01-19 3:22 ` Xia, Chenbo 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 4/9] vfio_user: implement DMA table and socket address API Chenbo Xia 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 5/9] vfio_user: implement interrupt related APIs Chenbo Xia 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 6/9] vfio_user: add client APIs of device attach/detach Chenbo Xia 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 7/9] vfio_user: add client APIs of DMA/IRQ/region Chenbo Xia 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 8/9] test/vfio_user: introduce functional test Chenbo Xia 2021-01-14 19:03 ` David Christensen 2021-01-19 3:27 ` Xia, Chenbo 2021-01-19 18:26 ` David Christensen 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 9/9] doc: add vfio-user library guide Chenbo Xia 2021-01-15 7:58 ` [dpdk-dev] [PATCH v2 0/9] Introduce vfio-user library David Marchand 2021-01-19 3:13 ` Xia, Chenbo
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20210114061411.39166-4-chenbo.xia@intel.com \ --to=chenbo.xia@intel.com \ --cc=beilei.xing@intel.com \ --cc=cunming.liang@intel.com \ --cc=david.marchand@redhat.com \ --cc=dev@dpdk.org \ --cc=jingjing.wu@intel.com \ --cc=miao.li@intel.com \ --cc=stephen@networkplumber.org \ --cc=thomas@monjalon.net \ --cc=xiuchun.lu@intel.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: link
DPDK patches and discussions This inbox may be cloned and mirrored by anyone: git clone --mirror https://inbox.dpdk.org/dev/0 dev/git/0.git # If you have public-inbox 1.1+ installed, you may # initialize and index your mirror using the following commands: public-inbox-init -V2 dev dev/ https://inbox.dpdk.org/dev \ dev@dpdk.org public-inbox-index dev Example config snippet for mirrors. Newsgroup available over NNTP: nntp://inbox.dpdk.org/inbox.dpdk.dev AGPL code for this site: git clone https://public-inbox.org/public-inbox.git