From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id C2B69A0542 for ; Thu, 24 Nov 2022 14:37:46 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 9D66B42DBD; Thu, 24 Nov 2022 14:37:46 +0100 (CET) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id BFFD442DB6 for ; Thu, 24 Nov 2022 14:37:44 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1669297064; x=1700833064; h=from:to:cc:subject:date:message-id:mime-version: content-transfer-encoding; bh=jOv8gLlhTBxyG+9VXTFvIQ4J6QkowTt1CZPyl4j+5Mk=; b=UjDH6hxBA+btsPcaUOYwI0sX4e1trYkrusE/cPM8RlhD2vwi9ROsucmN 0HUeIuVZuWtMe9pn1M1zxbuJMr4m665W5I8edbAWPnLQ2elvfGI810NMe 4AbHN5AIpJeLEJx8lkTGWuKpC46hK1jRIHKa0YOCzudRWE2PQTI6GMaSm ngnGP8TbFSk+9z1d2uRfvAhBtKHAy9crG65Z2Idk36iucoca/V8CoFuqN aQLjws67fiR+Zogq7xLOgzr21Dst1Du/31aw7zMlCH5tuC4OckB7AvcBK 1PAgVlK6iaDm3Z5iTjoeJbX2n8v9bI8466kPN6x8X+KEQRI2sLYEAJhES w==; X-IronPort-AV: E=McAfee;i="6500,9779,10541"; a="311939258" X-IronPort-AV: E=Sophos;i="5.96,190,1665471600"; d="scan'208";a="311939258" Received: from fmsmga007.fm.intel.com ([10.253.24.52]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Nov 2022 05:37:43 -0800 X-IronPort-AV: E=McAfee;i="6500,9779,10541"; a="644495597" X-IronPort-AV: E=Sophos;i="5.96,190,1665471600"; d="scan'208";a="644495597" Received: from unknown (HELO dpdkserver..) ([10.239.252.174]) by fmsmga007-auth.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Nov 2022 05:37:42 -0800 From: Yiding Zhou To: stable@dpdk.org Cc: ktraynor@redhat.com, Yiding Zhou Subject: [PATCH 21.11.3] net/iavf: add thread for event callbacks Date: Thu, 24 Nov 2022 21:41:46 +0800 Message-Id: <20221124134146.855000-1-yidingx.zhou@intel.com> X-Mailer: git-send-email 2.34.1 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: stable@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: patches for DPDK stable branches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: stable-bounces@dpdk.org [upstream commit cb5c1b91f76f436724cd09f26c7432b2775b519c] All callbacks registered for ethdev events are called in eal-intr-thread, and some of them execute virtchnl commands. Because interrupts are disabled in the intr thread, there will be no response received for these commands. So all callbacks should be called in a new context. When the device is bonded, the bond pmd registers callback for LSC event to execute virtchnl commands to reinitialize the device, it would also raise the above issue. This commit add a new thread to call all event callbacks. Fixes: 48de41ca11f0 ("net/avf: enable link status update") Fixes: 84108425054a ("net/iavf: support asynchronous virtual channel message") Signed-off-by: Yiding Zhou --- drivers/net/iavf/iavf.h | 2 + drivers/net/iavf/iavf_ethdev.c | 5 ++ drivers/net/iavf/iavf_vchnl.c | 153 +++++++++++++++++++++++++++++++-- 3 files changed, 154 insertions(+), 6 deletions(-) diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h index 29692e3994..afffc1a13e 100644 --- a/drivers/net/iavf/iavf.h +++ b/drivers/net/iavf/iavf.h @@ -400,6 +400,8 @@ _atomic_set_async_response_cmd(struct iavf_info *vf, enum virtchnl_ops ops) } int iavf_check_api_version(struct iavf_adapter *adapter); int iavf_get_vf_resource(struct iavf_adapter *adapter); +void iavf_dev_event_handler_fini(void); +int iavf_dev_event_handler_init(void); void iavf_handle_virtchnl_msg(struct rte_eth_dev *dev); int iavf_enable_vlan_strip(struct iavf_adapter *adapter); int iavf_disable_vlan_strip(struct iavf_adapter *adapter); diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c index f835457e4f..11ddd6dc16 100644 --- a/drivers/net/iavf/iavf_ethdev.c +++ b/drivers/net/iavf/iavf_ethdev.c @@ -2561,6 +2561,9 @@ iavf_dev_init(struct rte_eth_dev *eth_dev) rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr, ð_dev->data->mac_addrs[0]); + if (iavf_dev_event_handler_init()) + goto init_vf_err; + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { /* register callback func to eal lib */ rte_intr_callback_register(pci_dev->intr_handle, @@ -2715,6 +2718,8 @@ iavf_dev_uninit(struct rte_eth_dev *dev) iavf_dev_close(dev); + iavf_dev_event_handler_fini(); + return 0; } diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c index 1bd3559ec2..d408e19adc 100644 --- a/drivers/net/iavf/iavf_vchnl.c +++ b/drivers/net/iavf/iavf_vchnl.c @@ -2,6 +2,7 @@ * Copyright(c) 2017 Intel Corporation */ +#include #include #include #include @@ -11,6 +12,7 @@ #include #include #include +#include #include #include @@ -27,6 +29,146 @@ #define MAX_TRY_TIMES 2000 #define ASQ_DELAY_MS 1 +#define MAX_EVENT_PENDING 16 + +struct iavf_event_element { + TAILQ_ENTRY(iavf_event_element) next; + struct rte_eth_dev *dev; + enum rte_eth_event_type event; + void *param; + size_t param_alloc_size; + uint8_t param_alloc_data[0]; +}; + +struct iavf_event_handler { + uint32_t ndev; + pthread_t tid; + int fd[2]; + pthread_mutex_t lock; + TAILQ_HEAD(event_lsit, iavf_event_element) pending; +}; + +static struct iavf_event_handler event_handler = { + .fd = {-1, -1}, +}; + +#ifndef TAILQ_FOREACH_SAFE +#define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = TAILQ_FIRST((head)); \ + (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) +#endif + +static void * +iavf_dev_event_handle(void *param __rte_unused) +{ + struct iavf_event_handler *handler = &event_handler; + TAILQ_HEAD(event_list, iavf_event_element) pending; + + while (true) { + char unused[MAX_EVENT_PENDING]; + ssize_t nr = read(handler->fd[0], &unused, sizeof(unused)); + if (nr <= 0) + break; + + TAILQ_INIT(&pending); + pthread_mutex_lock(&handler->lock); + TAILQ_CONCAT(&pending, &handler->pending, next); + pthread_mutex_unlock(&handler->lock); + + struct iavf_event_element *pos, *save_next; + TAILQ_FOREACH_SAFE(pos, &pending, next, save_next) { + TAILQ_REMOVE(&pending, pos, next); + rte_eth_dev_callback_process(pos->dev, pos->event, pos->param); + rte_free(pos); + } + } + + return NULL; +} + +static void +iavf_dev_event_post(struct rte_eth_dev *dev, + enum rte_eth_event_type event, + void *param, size_t param_alloc_size) +{ + struct iavf_event_handler *handler = &event_handler; + char notify_byte; + struct iavf_event_element *elem = rte_malloc(NULL, sizeof(*elem) + param_alloc_size, 0); + if (!elem) + return; + + elem->dev = dev; + elem->event = event; + elem->param = param; + elem->param_alloc_size = param_alloc_size; + if (param && param_alloc_size) { + rte_memcpy(elem->param_alloc_data, param, param_alloc_size); + elem->param = elem->param_alloc_data; + } + + pthread_mutex_lock(&handler->lock); + TAILQ_INSERT_TAIL(&handler->pending, elem, next); + pthread_mutex_unlock(&handler->lock); + + ssize_t nw = write(handler->fd[1], ¬ify_byte, 1); + RTE_SET_USED(nw); +} + +int +iavf_dev_event_handler_init(void) +{ + struct iavf_event_handler *handler = &event_handler; + + if (__atomic_add_fetch(&handler->ndev, 1, __ATOMIC_RELAXED) != 1) + return 0; +#if defined(RTE_EXEC_ENV_WINDOWS) && RTE_EXEC_ENV_WINDOWS != 0 + int err = _pipe(handler->fd, MAX_EVENT_PENDING, O_BINARY); +#else + int err = pipe(handler->fd); +#endif + if (err != 0) { + __atomic_sub_fetch(&handler->ndev, 1, __ATOMIC_RELAXED); + return -1; + } + + TAILQ_INIT(&handler->pending); + pthread_mutex_init(&handler->lock, NULL); + + if (rte_ctrl_thread_create(&handler->tid, "iavf-event-thread", + NULL, iavf_dev_event_handle, NULL)) { + __atomic_sub_fetch(&handler->ndev, 1, __ATOMIC_RELAXED); + return -1; + } + + return 0; +} + +void +iavf_dev_event_handler_fini(void) +{ + struct iavf_event_handler *handler = &event_handler; + + if (__atomic_sub_fetch(&handler->ndev, 1, __ATOMIC_RELAXED) != 0) + return; + + int unused = pthread_cancel(handler->tid); + RTE_SET_USED(unused); + close(handler->fd[0]); + close(handler->fd[1]); + handler->fd[0] = -1; + handler->fd[1] = -1; + + pthread_join(handler->tid, NULL); + pthread_mutex_destroy(&handler->lock); + + struct iavf_event_element *pos, *save_next; + TAILQ_FOREACH_SAFE(pos, &handler->pending, next, save_next) { + TAILQ_REMOVE(&handler->pending, pos, next); + rte_free(pos); + } +} + static uint32_t iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed) { @@ -278,8 +420,8 @@ iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg, case VIRTCHNL_EVENT_RESET_IMPENDING: PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event"); vf->vf_reset = true; - rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, - NULL); + iavf_dev_event_post(dev, RTE_ETH_EVENT_INTR_RESET, + NULL, 0); break; case VIRTCHNL_EVENT_LINK_CHANGE: PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event"); @@ -293,7 +435,7 @@ iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg, vf->link_speed = iavf_convert_link_speed(speed); } iavf_dev_link_update(dev, 0); - rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); + iavf_dev_event_post(dev, RTE_ETH_EVENT_INTR_LSC, NULL, 0); break; case VIRTCHNL_EVENT_PF_DRIVER_CLOSE: PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event"); @@ -359,9 +501,8 @@ iavf_handle_virtchnl_msg(struct rte_eth_dev *dev) desc.subtype = RTE_ETH_EVENT_IPSEC_UNKNOWN; desc.metadata = ev->ipsec_event_data; - rte_eth_dev_callback_process(dev, - RTE_ETH_EVENT_IPSEC, - &desc); + iavf_dev_event_post(dev, RTE_ETH_EVENT_IPSEC, + &desc, sizeof(desc)); return; } -- 2.34.1