From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from sesbmg22.ericsson.net (sesbmg22.ericsson.net [193.180.251.48]) by dpdk.org (Postfix) with ESMTP id 78CA62A66 for ; Wed, 7 Dec 2016 13:46:54 +0100 (CET) X-AuditID: c1b4fb30-c4bff700000054c8-c2-584804bc0e5c Received: from ESESSHC013.ericsson.se (Unknown_Domain [153.88.183.57]) by (Symantec Mail Security) with SMTP id 78.E5.21704.CB408485; Wed, 7 Dec 2016 13:46:53 +0100 (CET) Received: from cloudvm.ki.sw.ericsson.se (153.88.183.153) by smtps.internal.ericsson.com (153.88.183.57) with Microsoft SMTP Server (TLS) id 14.3.319.2; Wed, 7 Dec 2016 13:46:52 +0100 From: Jan Wickbom To: CC: , , Jan Wickbom Date: Wed, 7 Dec 2016 13:46:30 +0100 Message-ID: <1481114790-27379-1-git-send-email-jan.wickbom@ericsson.com> X-Mailer: git-send-email 1.9.1 In-Reply-To: <1480606010-6132-1-git-send-email-jan.wickbom@ericsson.com> References: <1480606010-6132-1-git-send-email-jan.wickbom@ericsson.com> MIME-Version: 1.0 Content-Type: text/plain X-Originating-IP: [153.88.183.153] X-Brightmail-Tracker: H4sIAAAAAAAAA+NgFlrOLMWRmVeSWpSXmKPExsUyM2K7pe5eFo8Ig43XmS3efdrOZHF9wgVW ByaPXwuWsnrMOxkYwBTFZZOSmpNZllqkb5fAlbGs5Q9zwWHbimev5rA2MN7V62Lk5JAQMJFY 0/iFqYuRi0NIYB2jxLwrS9ghnN2MEoe2P2EFqWIT0JFY/bYDzBYRUJTYuXcjE4jNLBAv8fXm HSCbg0NYwEpi2v9EkDCLgIrE84tHwUp4BTwkluw5ygixTE7i5LHJYGM4geIrplxhA7GFBNwl bl1vYoWoF5Q4OfMJC8R4CYmDL14wQ9SoSjxf2MAEMUdJ4vq86ywTGAVmIWmZhaRlASPTKkbR 4tTipNx0IyO91KLM5OLi/Dy9vNSSTYzA4Du45bfBDsaXzx0PMQpwMCrx8BZccosQYk0sK67M PcQowcGsJML7+r97hBBvSmJlVWpRfnxRaU5q8SFGaQ4WJXFes5X3w4UE0hNLUrNTUwtSi2Cy TBycUg2MzilPZtxZK/v9I7vXlSCpgBaRtD+X1SJjH+o4z+P0mtxjEbRtXozrdjal/2cTLQtk Dlt3PvLZ4vvy1qOEbanWk1U4l99ysz5+xMlCL9Xg0/kX1cfW7Jz3Oah2gvez64w1Wo/5ZN5P +8Sf9tDssav7BZYQpn17bTWVHlXabUvdffzzHdNbrxQ+KrEUZyQaajEXFScCAIgcdRo6AgAA Subject: [dpdk-dev] [PATCH v2] vhost: allow for many vhost user ports X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 07 Dec 2016 12:46:54 -0000 Currently select() is used to monitor file descriptors for vhostuser ports. This limits the number of ports possible to create since the fd number is used as index in the fd_set and we have seen fds > 1023. This patch changes select() to poll(). This way we can keep an packed (pollfd) array for the fds, e.g. as many fds as the size of the array. Also see: http://dpdk.org/ml/archives/dev/2016-April/037024.html Signed-off-by: Jan Wickbom Reported-by: Patrik Andersson --- v2: * removed unnecessary casts * static array replacing allocated memory lib/librte_vhost/fd_man.c | 142 +++++++++++++++++++++++++++------------------- lib/librte_vhost/fd_man.h | 2 +- 2 files changed, 85 insertions(+), 59 deletions(-) diff --git a/lib/librte_vhost/fd_man.c b/lib/librte_vhost/fd_man.c index 2d3eeb7..3c743e3 100644 --- a/lib/librte_vhost/fd_man.c +++ b/lib/librte_vhost/fd_man.c @@ -35,16 +35,43 @@ #include #include #include -#include #include #include +#include #include +#include #include +#include #include #include "fd_man.h" +#define FDPOLLERR (POLLERR | POLLHUP | POLLNVAL) + + +static struct pollfd rwfds[MAX_FDS]; + +/** + * Adjusts the highest index populated in the array of fds + * @return + * The new size of fdset. + */ +static int +fdset_shrink(struct fdset *pfdset) +{ + int idx; + + for (idx = pfdset->num - 1; + idx >= 0 && pfdset->fd[idx].fd == -1; + idx--) + ; + + pfdset->num = idx + 1; + + return pfdset->num; +} + /** * Returns the index in the fdset for a given fd. * If fd is -1, it means to search for a free entry. @@ -56,72 +83,62 @@ { int i; - if (pfdset == NULL) - return -1; - - for (i = 0; i < MAX_FDS && pfdset->fd[i].fd != fd; i++) + for (i = 0; i < pfdset->num && pfdset->fd[i].fd != fd; i++) ; - return i == MAX_FDS ? -1 : i; + return i == pfdset->num ? -1 : i; } static int fdset_find_free_slot(struct fdset *pfdset) { - return fdset_find_fd(pfdset, -1); + if (pfdset->num < MAX_FDS) + return pfdset->num; + else + return fdset_find_fd(pfdset, -1); } -static int -fdset_add_fd(struct fdset *pfdset, int idx, int fd, +static void +fdset_add_fd(struct fdset *pfdset, int idx, int fd, fd_cb rcb, fd_cb wcb, void *dat) { struct fdentry *pfdentry; - if (pfdset == NULL || idx >= MAX_FDS || fd >= FD_SETSIZE) - return -1; - pfdentry = &pfdset->fd[idx]; pfdentry->fd = fd; pfdentry->rcb = rcb; pfdentry->wcb = wcb; pfdentry->dat = dat; - - return 0; } /** - * Fill the read/write fd_set with the fds in the fdset. + * Compact the fdset and fill the read/write fds with the fds in the fdset. * @return - * the maximum fds filled in the read/write fd_set. + * the number of fds filled in the read/write fds. */ static int -fdset_fill(fd_set *rfset, fd_set *wfset, struct fdset *pfdset) +fdset_fill(struct pollfd *rwfds, struct fdset *pfdset) { struct fdentry *pfdentry; - int i, maxfds = -1; - int num = MAX_FDS; - - if (pfdset == NULL) - return -1; + int i; + int num; - for (i = 0; i < num; i++) { + for (i = 0, num = pfdset->num; i < num; i++) { pfdentry = &pfdset->fd[i]; - if (pfdentry->fd != -1) { - int added = 0; - if (pfdentry->rcb && rfset) { - FD_SET(pfdentry->fd, rfset); - added = 1; - } - if (pfdentry->wcb && wfset) { - FD_SET(pfdentry->fd, wfset); - added = 1; - } - if (added) - maxfds = pfdentry->fd < maxfds ? - maxfds : pfdentry->fd; + + if (pfdentry->fd < 0) { + /* Hole in the list. Move the last one here */ + + *pfdentry = pfdset->fd[num - 1]; + pfdset->fd[num - 1].fd = -1; + num = fdset_shrink(pfdset); } + rwfds[i].fd = pfdentry->fd; + rwfds[i].events = pfdentry->rcb ? POLLIN : 0; + rwfds[i].events |= pfdentry->wcb ? POLLOUT : 0; } - return maxfds; + + return i; } void @@ -132,6 +149,8 @@ if (pfdset == NULL) return; + pthread_mutex_init(&pfdset->fd_mutex, NULL); + for (i = 0; i < MAX_FDS; i++) { pfdset->fd[i].fd = -1; pfdset->fd[i].dat = NULL; @@ -152,14 +171,15 @@ pthread_mutex_lock(&pfdset->fd_mutex); - /* Find a free slot in the list. */ i = fdset_find_free_slot(pfdset); - if (i == -1 || fdset_add_fd(pfdset, i, fd, rcb, wcb, dat) < 0) { + if (i == -1) { pthread_mutex_unlock(&pfdset->fd_mutex); return -2; } - pfdset->num++; + fdset_add_fd(pfdset, i, fd, rcb, wcb, dat); + if (i == pfdset->num) + pfdset->num++; pthread_mutex_unlock(&pfdset->fd_mutex); @@ -189,7 +209,7 @@ pfdset->fd[i].fd = -1; pfdset->fd[i].rcb = pfdset->fd[i].wcb = NULL; pfdset->fd[i].dat = NULL; - pfdset->num--; + fdset_shrink(pfdset); i = -1; } pthread_mutex_unlock(&pfdset->fd_mutex); @@ -211,12 +231,12 @@ pfdset->fd[index].fd = -1; pfdset->fd[index].rcb = pfdset->fd[index].wcb = NULL; - pfdset->fd[index].dat = NULL; - pfdset->num--; + fdset_shrink(pfdset); pthread_mutex_unlock(&pfdset->fd_mutex); } + /** * This functions runs in infinite blocking loop until there is no fd in * pfdset. It calls corresponding r/w handler if there is event on the fd. @@ -229,44 +249,47 @@ void fdset_event_dispatch(struct fdset *pfdset) { - fd_set rfds, wfds; - int i, maxfds; + int i; struct fdentry *pfdentry; - int num = MAX_FDS; + int numfds; fd_cb rcb, wcb; void *dat; int fd; int remove1, remove2; int ret; + int handled; if (pfdset == NULL) return; + memset(rwfds, 0, sizeof(rwfds)); + while (1) { - struct timeval tv; - tv.tv_sec = 1; - tv.tv_usec = 0; - FD_ZERO(&rfds); - FD_ZERO(&wfds); pthread_mutex_lock(&pfdset->fd_mutex); - maxfds = fdset_fill(&rfds, &wfds, pfdset); + numfds = fdset_fill(rwfds, pfdset); pthread_mutex_unlock(&pfdset->fd_mutex); /* - * When select is blocked, other threads might unregister + * When poll is blocked, other threads might unregister * listenfds from and register new listenfds into fdset. - * When select returns, the entries for listenfds in the fdset + * When poll returns, the entries for listenfds in the fdset * might have been updated. It is ok if there is unwanted call * for new listenfds. */ - ret = select(maxfds + 1, &rfds, &wfds, NULL, &tv); + ret = poll(rwfds, numfds, 1000 /* millisecs */); + if (ret <= 0) continue; - for (i = 0; i < num; i++) { + for (i = handled = 0; i < numfds && handled < ret; i++) { + if (!rwfds[i].revents) + continue; + + handled++; remove1 = remove2 = 0; + pthread_mutex_lock(&pfdset->fd_mutex); pfdentry = &pfdset->fd[i]; fd = pfdentry->fd; @@ -275,9 +298,12 @@ dat = pfdentry->dat; pfdentry->busy = 1; pthread_mutex_unlock(&pfdset->fd_mutex); - if (fd >= 0 && FD_ISSET(fd, &rfds) && rcb) + + if (fd >= 0 && rcb && + rwfds[i].revents & (POLLIN | FDPOLLERR)) rcb(fd, dat, &remove1); - if (fd >= 0 && FD_ISSET(fd, &wfds) && wcb) + if (fd >= 0 && wcb && + rwfds[i].revents & (POLLOUT | FDPOLLERR)) wcb(fd, dat, &remove2); pfdentry->busy = 0; /* diff --git a/lib/librte_vhost/fd_man.h b/lib/librte_vhost/fd_man.h index bd66ed1..b5ba688 100644 --- a/lib/librte_vhost/fd_man.h +++ b/lib/librte_vhost/fd_man.h @@ -51,7 +51,7 @@ struct fdentry { struct fdset { struct fdentry fd[MAX_FDS]; pthread_mutex_t fd_mutex; - int num; /* current fd number of this fdset */ + int num; /* highest index occupied in fd array + 1 */ }; -- 1.9.1