From: Chenbo Xia <chenbo.xia@intel.com> To: dev@dpdk.org, thomas@monjalon.net, david.marchand@redhat.com Cc: stephen@networkplumber.org, cunming.liang@intel.com, xiuchun.lu@intel.com, miao.li@intel.com, jingjing.wu@intel.com Subject: [dpdk-dev] [PATCH 5/9] vfio_user: implement interrupt related APIs Date: Fri, 18 Dec 2020 15:38:47 +0800 Message-ID: <20201218073851.93609-6-chenbo.xia@intel.com> (raw) In-Reply-To: <20201218073851.93609-1-chenbo.xia@intel.com> This patch implements two interrupt related APIs, which are rte_vfio_user_get_irq() and rte_vfio_user_set_irq_info(). The former is for devices to get interrupt configuration (e.g., irqfds). The latter is for setting interrupt information before vfio-user starts. Signed-off-by: Chenbo Xia <chenbo.xia@intel.com> Signed-off-by: Xiuchun Lu <xiuchun.lu@intel.com> --- lib/librte_vfio_user/rte_vfio_user.h | 44 ++++ lib/librte_vfio_user/version.map | 2 + lib/librte_vfio_user/vfio_user_base.h | 8 + lib/librte_vfio_user/vfio_user_server.c | 292 +++++++++++++++++++++++- lib/librte_vfio_user/vfio_user_server.h | 6 + 5 files changed, 347 insertions(+), 5 deletions(-) diff --git a/lib/librte_vfio_user/rte_vfio_user.h b/lib/librte_vfio_user/rte_vfio_user.h index 044c43e7dc..6c12b0b6ed 100644 --- a/lib/librte_vfio_user/rte_vfio_user.h +++ b/lib/librte_vfio_user/rte_vfio_user.h @@ -69,6 +69,11 @@ struct rte_vfio_user_regions { struct rte_vfio_user_reg_info reg_info[]; }; +struct rte_vfio_user_irq_info { + uint32_t irq_num; + struct vfio_irq_info irq_info[]; +}; + /** * Below APIs are for vfio-user server (device provider) to use: * *rte_vfio_user_register @@ -76,8 +81,10 @@ struct rte_vfio_user_regions { * *rte_vfio_user_start * *rte_vfio_get_sock_addr * *rte_vfio_user_get_mem_table + * *rte_vfio_user_get_irq * *rte_vfio_user_set_dev_info * *rte_vfio_user_set_reg_info + * *rte_vfio_user_set_irq_info */ /** @@ -181,4 +188,41 @@ int rte_vfio_user_set_reg_info(const char *sock_addr, __rte_experimental int rte_vfio_get_sock_addr(int dev_id, char *buf, size_t len); +/** + * Get the irqfds of a vfio-user device. + * + * @param dev_id + * Vfio-user device ID + * @param index + * irq index + * @param count + * irq count + * @param[out] fds + * Pointer to the irqfds + * @return + * 0 on success, -1 on failure + */ +__rte_experimental +int rte_vfio_user_get_irq(int dev_id, uint32_t index, uint32_t count, + int *fds); + +/** + * Set the irq information for a vfio-user device. + * + * This information must be set before calling rte_vfio_user_start, and should + * not be updated after start. Update after start can be done by unregistration + * and re-registration, and then the device-level change can be detected by + * vfio-user client. + * + * @param sock_addr + * Unix domain socket address + * @param irq + * IRQ information for the vfio-user device + * @return + * 0 on success, -1 on failure + */ +__rte_experimental +int rte_vfio_user_set_irq_info(const char *sock_addr, + struct rte_vfio_user_irq_info *irq); + #endif diff --git a/lib/librte_vfio_user/version.map b/lib/librte_vfio_user/version.map index 3a50b5ef0e..621a51a9fc 100644 --- a/lib/librte_vfio_user/version.map +++ b/lib/librte_vfio_user/version.map @@ -6,8 +6,10 @@ EXPERIMENTAL { rte_vfio_user_start; rte_vfio_get_sock_addr; rte_vfio_user_get_mem_table; + rte_vfio_user_get_irq; rte_vfio_user_set_dev_info; rte_vfio_user_set_reg_info; + rte_vfio_user_set_irq_info; local: *; }; diff --git a/lib/librte_vfio_user/vfio_user_base.h b/lib/librte_vfio_user/vfio_user_base.h index 5f5e651e87..7fed52a44e 100644 --- a/lib/librte_vfio_user/vfio_user_base.h +++ b/lib/librte_vfio_user/vfio_user_base.h @@ -61,6 +61,12 @@ struct vfio_user_reg { uint8_t rsvd[VFIO_USER_MAX_RSVD]; }; +struct vfio_user_irq_set { + struct vfio_irq_set set; + /* Reserved for data of irq set */ + uint8_t rsvd[VFIO_USER_MAX_RSVD]; +}; + struct vfio_user_reg_rw { uint64_t reg_offset; uint32_t reg_idx; @@ -83,6 +89,8 @@ typedef struct vfio_user_msg { struct rte_vfio_user_mem_reg memory[VFIO_USER_MSG_MAX_NREG]; struct vfio_device_info dev_info; struct vfio_user_reg reg_info; + struct vfio_irq_info irq_info; + struct vfio_user_irq_set irq_set; struct vfio_user_reg_rw reg_rw; } payload; int fds[VFIO_USER_MAX_FD]; diff --git a/lib/librte_vfio_user/vfio_user_server.c b/lib/librte_vfio_user/vfio_user_server.c index 1162e463b7..cbaf3b5ed5 100644 --- a/lib/librte_vfio_user/vfio_user_server.c +++ b/lib/librte_vfio_user/vfio_user_server.c @@ -9,6 +9,7 @@ #include <sys/socket.h> #include <sys/mman.h> #include <sys/un.h> +#include <sys/eventfd.h> #include "vfio_user_server.h" @@ -301,6 +302,146 @@ static int vfio_user_device_get_reg_info(struct vfio_user_server *dev, return 0; } +static int vfio_user_device_get_irq_info(struct vfio_user_server *dev, + VFIO_USER_MSG *msg) +{ + struct vfio_irq_info *irq_info = &msg->payload.irq_info; + struct rte_vfio_user_irq_info *info = dev->irqs.info; + uint32_t i; + + if (vfio_user_check_msg_fdnum(msg, 0) != 0) + return -EINVAL; + + for (i = 0; i < info->irq_num; i++) { + if (irq_info->index == info->irq_info[i].index) { + irq_info->count = info->irq_info[i].count; + irq_info->flags |= info->irq_info[i].flags; + break; + } + } + if (i == info->irq_num) + return -EINVAL; + + VFIO_USER_LOG(DEBUG, "IRQ info: argsz(0x%x), flags(0x%x), index(0x%x)," + " count(0x%x)\n", irq_info->argsz, irq_info->flags, + irq_info->index, irq_info->count); + + return 0; +} + +static inline int irq_set_trigger(struct vfio_user_irqs *irqs, + struct vfio_irq_set *irq_set, VFIO_USER_MSG *msg) +{ + uint32_t i = irq_set->start; + int eventfd; + + switch (irq_set->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { + case VFIO_IRQ_SET_DATA_NONE: + if (vfio_user_check_msg_fdnum(msg, 0) != 0) + return -EINVAL; + + for (; i < irq_set->start + irq_set->count; i++) { + eventfd = irqs->fds[irq_set->index][i]; + if (eventfd >= 0) { + if (eventfd_write(eventfd, (eventfd_t)1)) + return -errno; + } + } + break; + case VFIO_IRQ_SET_DATA_BOOL: + if (vfio_user_check_msg_fdnum(msg, 0) != 0) + return -EINVAL; + + uint8_t *idx = irq_set->data; + for (; i < irq_set->start + irq_set->count; i++, idx++) { + eventfd = irqs->fds[irq_set->index][i]; + if (eventfd >= 0 && *idx == 1) { + if (eventfd_write(eventfd, (eventfd_t)1)) + return -errno; + } + } + break; + case VFIO_IRQ_SET_DATA_EVENTFD: + if (vfio_user_check_msg_fdnum(msg, irq_set->count) != 0) + return -EINVAL; + + int32_t *fds = msg->fds; + for (; i < irq_set->start + irq_set->count; i++, fds++) { + eventfd = irqs->fds[irq_set->index][i]; + if (eventfd >= 0) + close(eventfd); /* Clear original irqfd*/ + if (*fds >= 0) + irqs->fds[irq_set->index][i] = *fds; + } + break; + default: + return -EINVAL; + } + + return 0; +} + +static void vfio_user_disable_irqs(struct vfio_user_irqs *irqs) +{ + struct rte_vfio_user_irq_info *info = irqs->info; + uint32_t i, j; + + for (i = 0; i < info->irq_num; i++) { + for (j = 0; j < info->irq_info[i].count; j++) { + if (irqs->fds[i][j] != -1) { + close(irqs->fds[i][j]); + irqs->fds[i][j] = -1; + } + } + } +} + +static int vfio_user_device_set_irqs(struct vfio_user_server *dev, + VFIO_USER_MSG *msg) +{ + struct vfio_user_irq_set *irq = &msg->payload.irq_set; + struct vfio_irq_set *irq_set = &irq->set; + struct rte_vfio_user_irq_info *info = dev->irqs.info; + int ret = 0; + + if (info->irq_num <= irq_set->index + || info->irq_info[irq_set->index].count < + irq_set->start + irq_set->count) { + vfio_user_close_msg_fds(msg); + return -EINVAL; + } + + if (irq_set->count == 0) { + if (irq_set->flags & VFIO_IRQ_SET_DATA_NONE) { + vfio_user_disable_irqs(&dev->irqs); + return 0; + } + vfio_user_close_msg_fds(msg); + return -EINVAL; + } + + switch (irq_set->flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { + /* Mask/Unmask not supported for now */ + case VFIO_IRQ_SET_ACTION_MASK: + /* FALLTHROUGH */ + case VFIO_IRQ_SET_ACTION_UNMASK: + return 0; + case VFIO_IRQ_SET_ACTION_TRIGGER: + ret = irq_set_trigger(&dev->irqs, irq_set, msg); + break; + default: + return -EINVAL; + } + + VFIO_USER_LOG(DEBUG, "Set IRQ: argsz(0x%x), flags(0x%x), index(0x%x), " + "start(0x%x), count(0x%x)\n", irq_set->argsz, irq_set->flags, + irq_set->index, irq_set->start, irq_set->count); + + /* Do not reply fds back */ + msg->fd_num = 0; + return ret; +} + static int vfio_user_region_read(struct vfio_user_server *dev, VFIO_USER_MSG *msg) { @@ -408,6 +549,48 @@ static inline void vfio_user_destroy_mem(struct vfio_user_server *dev) dev->mem = NULL; } +static inline void vfio_user_destroy_irq(struct vfio_user_server *dev) +{ + struct vfio_user_irqs *irq = &dev->irqs; + int *fd; + uint32_t i, j; + + if (!irq->info) + return; + + for (i = 0; i < irq->info->irq_num; i++) { + fd = irq->fds[i]; + + for (j = 0; j < irq->info->irq_info[i].count; j++) { + if (fd[j] != -1) + close(fd[j]); + } + + free(fd); + } + + free(irq->fds); +} + +static inline void vfio_user_clean_irqfd(struct vfio_user_server *dev) +{ + struct vfio_user_irqs *irq = &dev->irqs; + int *fd; + uint32_t i, j; + + if (!irq->info) + return; + + for (i = 0; i < irq->info->irq_num; i++) { + fd = irq->fds[i]; + + for (j = 0; j < irq->info->irq_info[i].count; j++) { + close(fd[j]); + fd[j] = -1; + } + } +} + static int vfio_user_device_reset(struct vfio_user_server *dev, VFIO_USER_MSG *msg) { @@ -422,6 +605,7 @@ static int vfio_user_device_reset(struct vfio_user_server *dev, return -ENOTSUP; vfio_user_destroy_mem_entries(dev->mem); + vfio_user_clean_irqfd(dev); dev->is_ready = 0; if (dev->ops->reset_device) @@ -437,8 +621,8 @@ static vfio_user_msg_handler_t vfio_user_msg_handlers[VFIO_USER_MAX] = { [VFIO_USER_DMA_UNMAP] = vfio_user_dma_unmap, [VFIO_USER_DEVICE_GET_INFO] = vfio_user_device_get_info, [VFIO_USER_DEVICE_GET_REGION_INFO] = vfio_user_device_get_reg_info, - [VFIO_USER_DEVICE_GET_IRQ_INFO] = NULL, - [VFIO_USER_DEVICE_SET_IRQS] = NULL, + [VFIO_USER_DEVICE_GET_IRQ_INFO] = vfio_user_device_get_irq_info, + [VFIO_USER_DEVICE_SET_IRQS] = vfio_user_device_set_irqs, [VFIO_USER_REGION_READ] = vfio_user_region_read, [VFIO_USER_REGION_WRITE] = vfio_user_region_write, [VFIO_USER_DMA_READ] = NULL, @@ -829,6 +1013,7 @@ static int vfio_user_message_handler(int dev_id, int fd) * to avoid errors in data path handling */ if ((cmd == VFIO_USER_DMA_MAP || cmd == VFIO_USER_DMA_UNMAP || + cmd == VFIO_USER_DEVICE_SET_IRQS || cmd == VFIO_USER_DEVICE_RESET) && dev->ops->lock_dp) { dev->ops->lock_dp(dev_id, 1); @@ -871,7 +1056,8 @@ static int vfio_user_message_handler(int dev_id, int fd) if (vfio_user_is_ready(dev) && dev->ops->new_device) dev->ops->new_device(dev_id); } else { - if ((cmd == VFIO_USER_DMA_MAP || cmd == VFIO_USER_DMA_UNMAP) + if ((cmd == VFIO_USER_DMA_MAP || cmd == VFIO_USER_DMA_UNMAP + || cmd == VFIO_USER_DEVICE_SET_IRQS) && dev->ops->update_status) dev->ops->update_status(dev_id); } @@ -898,6 +1084,7 @@ static int vfio_user_sock_read(int fd, void *data) if (dev) { dev->ops->destroy_device(dev_id); vfio_user_destroy_mem_entries(dev->mem); + vfio_user_clean_irqfd(dev); dev->is_ready = 0; dev->msg_id = 0; } @@ -995,9 +1182,9 @@ vfio_user_start_server(struct vfio_user_server_socket *sk) } /* All the info must be set before start */ - if (!dev->dev_info || !dev->reg) { + if (!dev->dev_info || !dev->reg || !dev->irqs.info) { VFIO_USER_LOG(ERR, "Failed to start, " - "dev/reg info must be set before start\n"); + "dev/reg/irq info must be set before start\n"); return -1; } @@ -1109,6 +1296,7 @@ int rte_vfio_user_unregister(const char *sock_addr) return -1; } vfio_user_destroy_mem(dev); + vfio_user_destroy_irq(dev); vfio_user_del_device(dev); return 0; @@ -1269,3 +1457,97 @@ const struct rte_vfio_user_mem *rte_vfio_user_get_mem_table(int dev_id) return dev->mem; } + +int rte_vfio_user_get_irq(int dev_id, uint32_t index, uint32_t count, int *fds) +{ + struct vfio_user_server *dev; + struct vfio_user_irqs *irqs; + uint32_t irq_max; + + dev = vfio_user_get_device(dev_id); + if (!dev) { + VFIO_USER_LOG(ERR, "Failed to get irq info:" + "device %d not found.\n", dev_id); + return -1; + } + + if (!fds) + return -1; + + irqs = &dev->irqs; + if (index >= irqs->info->irq_num) + return -1; + + irq_max = irqs->info->irq_info[index].count; + if (count > irq_max) + return -1; + + memcpy(fds, dev->irqs.fds[index], count * sizeof(int)); + return 0; +} + +int rte_vfio_user_set_irq_info(const char *sock_addr, + struct rte_vfio_user_irq_info *irq) +{ + struct vfio_user_server *dev; + struct vfio_user_server_socket *sk; + uint32_t i; + int dev_id, ret; + + if (!irq) + return -1; + + pthread_mutex_lock(&vfio_ep_sock.mutex); + sk = find_vfio_user_socket(sock_addr); + pthread_mutex_unlock(&vfio_ep_sock.mutex); + + if (!sk) { + VFIO_USER_LOG(ERR, "Failed to set irq info with sock_addr:" + "%s: addr not registered.\n", sock_addr); + return -1; + } + + dev_id = sk->sock.dev_id; + dev = vfio_user_get_device(dev_id); + if (!dev) { + VFIO_USER_LOG(ERR, "Failed to set irq info:" + "device %d not found.\n", dev_id); + return -1; + } + + if (dev->started) { + VFIO_USER_LOG(ERR, "Failed to set irq info for device %d\n" + ", device already started\n", dev_id); + return -1; + } + + if (!dev->irqs.info) + vfio_user_destroy_irq(dev); + + dev->irqs.info = irq; + + dev->irqs.fds = malloc(irq->irq_num * sizeof(int *)); + if (!dev->irqs.fds) + return -1; + + for (i = 0; i < irq->irq_num; i++) { + uint32_t sz = irq->irq_info[i].count * sizeof(int); + dev->irqs.fds[i] = malloc(sz); + if (!dev->irqs.fds[i]) { + ret = -1; + goto exit; + } + + memset(dev->irqs.fds[i], 0xFF, sz); + } + + return 0; +exit: + for (--i;; i--) { + free(dev->irqs.fds[i]); + if (i == 0) + break; + } + free(dev->irqs.fds); + return ret; +} diff --git a/lib/librte_vfio_user/vfio_user_server.h b/lib/librte_vfio_user/vfio_user_server.h index 737940de62..f357c59705 100644 --- a/lib/librte_vfio_user/vfio_user_server.h +++ b/lib/librte_vfio_user/vfio_user_server.h @@ -9,6 +9,11 @@ #include "vfio_user_base.h" +struct vfio_user_irqs { + struct rte_vfio_user_irq_info *info; + int **fds; +}; + struct vfio_user_server { int dev_id; int is_ready; @@ -21,6 +26,7 @@ struct vfio_user_server { struct rte_vfio_user_mem *mem; struct vfio_device_info *dev_info; struct rte_vfio_user_regions *reg; + struct vfio_user_irqs irqs; }; typedef int (*event_handler)(int fd, void *data); -- 2.17.1
next prev parent reply other threads:[~2020-12-18 7:55 UTC|newest] Thread overview: 43+ messages / expand[flat|nested] mbox.gz Atom feed top 2020-12-18 7:38 [dpdk-dev] [PATCH 0/9] Introduce vfio-user library Chenbo Xia 2020-12-18 7:38 ` [dpdk-dev] [PATCH 1/9] lib: introduce " Chenbo Xia 2020-12-18 17:13 ` Stephen Hemminger 2020-12-19 6:12 ` Xia, Chenbo 2020-12-18 17:17 ` Stephen Hemminger 2020-12-19 6:25 ` Xia, Chenbo 2020-12-18 7:38 ` [dpdk-dev] [PATCH 2/9] vfio_user: implement lifecycle related APIs Chenbo Xia 2021-01-05 8:34 ` Xing, Beilei 2021-01-05 9:58 ` Xia, Chenbo 2020-12-18 7:38 ` [dpdk-dev] [PATCH 3/9] vfio_user: implement device and region " Chenbo Xia 2021-01-06 5:51 ` Xing, Beilei 2021-01-06 7:50 ` Xia, Chenbo 2020-12-18 7:38 ` [dpdk-dev] [PATCH 4/9] vfio_user: implement DMA table and socket address API Chenbo Xia 2020-12-18 7:38 ` Chenbo Xia [this message] 2020-12-30 1:04 ` [dpdk-dev] [PATCH 5/9] vfio_user: implement interrupt related APIs Wu, Jingjing 2020-12-30 2:31 ` Xia, Chenbo 2020-12-18 7:38 ` [dpdk-dev] [PATCH 6/9] vfio_user: add client APIs of device attach/detach Chenbo Xia 2020-12-18 7:38 ` [dpdk-dev] [PATCH 7/9] vfio_user: add client APIs of DMA/IRQ/region Chenbo Xia 2021-01-07 2:41 ` Xing, Beilei 2021-01-07 7:26 ` Xia, Chenbo 2020-12-18 7:38 ` [dpdk-dev] [PATCH 8/9] test/vfio_user: introduce functional test Chenbo Xia 2020-12-18 7:38 ` [dpdk-dev] [PATCH 9/9] doc: add vfio-user library guide Chenbo Xia 2021-01-06 5:07 ` Xing, Beilei 2021-01-06 7:43 ` Xia, Chenbo 2020-12-18 9:37 ` [dpdk-dev] [PATCH 0/9] Introduce vfio-user library David Marchand 2020-12-18 14:07 ` Thanos Makatos 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 " Chenbo Xia 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 1/9] lib: introduce " Chenbo Xia 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 2/9] vfio_user: implement lifecycle related APIs Chenbo Xia 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 3/9] vfio_user: implement device and region " Chenbo Xia 2021-01-14 18:48 ` David Christensen 2021-01-19 3:22 ` Xia, Chenbo 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 4/9] vfio_user: implement DMA table and socket address API Chenbo Xia 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 5/9] vfio_user: implement interrupt related APIs Chenbo Xia 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 6/9] vfio_user: add client APIs of device attach/detach Chenbo Xia 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 7/9] vfio_user: add client APIs of DMA/IRQ/region Chenbo Xia 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 8/9] test/vfio_user: introduce functional test Chenbo Xia 2021-01-14 19:03 ` David Christensen 2021-01-19 3:27 ` Xia, Chenbo 2021-01-19 18:26 ` David Christensen 2021-01-14 6:14 ` [dpdk-dev] [PATCH v2 9/9] doc: add vfio-user library guide Chenbo Xia 2021-01-15 7:58 ` [dpdk-dev] [PATCH v2 0/9] Introduce vfio-user library David Marchand 2021-01-19 3:13 ` Xia, Chenbo
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20201218073851.93609-6-chenbo.xia@intel.com \ --to=chenbo.xia@intel.com \ --cc=cunming.liang@intel.com \ --cc=david.marchand@redhat.com \ --cc=dev@dpdk.org \ --cc=jingjing.wu@intel.com \ --cc=miao.li@intel.com \ --cc=stephen@networkplumber.org \ --cc=thomas@monjalon.net \ --cc=xiuchun.lu@intel.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: link
DPDK patches and discussions This inbox may be cloned and mirrored by anyone: git clone --mirror https://inbox.dpdk.org/dev/0 dev/git/0.git # If you have public-inbox 1.1+ installed, you may # initialize and index your mirror using the following commands: public-inbox-init -V2 dev dev/ https://inbox.dpdk.org/dev \ dev@dpdk.org public-inbox-index dev Example config snippet for mirrors. Newsgroup available over NNTP: nntp://inbox.dpdk.org/inbox.dpdk.dev AGPL code for this site: git clone https://public-inbox.org/public-inbox.git