From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 1C9DDA00C4; Fri, 5 Aug 2022 13:00:14 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id C734940C35; Fri, 5 Aug 2022 13:00:13 +0200 (CEST) Received: from shelob.oktetlabs.ru (shelob.oktetlabs.ru [91.220.146.113]) by mails.dpdk.org (Postfix) with ESMTP id CB3C2400D5 for ; Fri, 5 Aug 2022 13:00:11 +0200 (CEST) Received: from [192.168.1.39] (unknown [188.170.75.116]) (using TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 (128/128 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)) (No client certificate requested) by shelob.oktetlabs.ru (Postfix) with ESMTPSA id A7EADA5; Fri, 5 Aug 2022 14:00:10 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 shelob.oktetlabs.ru A7EADA5 DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=oktetlabs.ru; s=default; t=1659697211; bh=ub4QhA9tKSSvczBZ7RP7a0WaSqRN7QXAIGISmWRr0YI=; h=Date:Subject:To:Cc:References:From:In-Reply-To:From; b=UarthrSH4wq1RGPBNjrKYGA6TdivnTtIVyR/RzMfxQvPQEtZbMEfqI2bu7BeZ3Nve gDW9nuRgTUgqO6hDZvTKRLLxsFEMavy5n3W6uvflKc+ImDnmpZjHQS43DkqRXsXqpL VDprRZX84p+zgBQoOrlm6Vr0v/QOJ1jjzBwN2Y34= Message-ID: <7a4a9f61-1117-6ef0-3f0a-36576c193cc9@oktetlabs.ru> Date: Fri, 5 Aug 2022 14:00:09 +0300 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.11.0 Subject: Re: [PATCH v5 04/12] net/nfp: add initial flower firmware support Content-Language: en-US To: Chaoyong He , dev@dpdk.org Cc: niklas.soderlund@corigine.com, Heinrich Kuhn References: <1659681155-16525-1-git-send-email-chaoyong.he@corigine.com> <1659681155-16525-5-git-send-email-chaoyong.he@corigine.com> From: Andrew Rybchenko In-Reply-To: <1659681155-16525-5-git-send-email-chaoyong.he@corigine.com> Content-Type: text/plain; charset=UTF-8; format=flowed Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org On 8/5/22 09:32, Chaoyong He wrote: > This commits adds the basic probing infrastructure to support the flower "This commits adds" -> "Add" It is the description of the commit from the very beginning. > firmware. This firmware is geared towards offloading OVS and can > generally be found in /lib/firmware/netronome/flower. It is also used by > the NFP kernel driver when OVS offload with TC is desired. > > This commit also adds the basic infrastructure needed by the flower Same here. > firmware to operate. The firmware requires threads to service both the > PF vNIC and the ctrl vNIC. The PF is responsible for handling any > fallback traffic and the ctrl vNIC is used to communicate OVS flows > and flow statistics to and from the smartNIC. rte_services are used to > facilitate this logic. > > This commit also adds the cpp service, used for some user tools. > > Signed-off-by: Chaoyong He > Signed-off-by: Heinrich Kuhn > Reviewed-by: Niklas Söderlund > --- > drivers/net/nfp/flower/nfp_flower.c | 101 ++++++++++++++++++++++++++++++++++++ > drivers/net/nfp/flower/nfp_flower.h | 22 ++++++++ > drivers/net/nfp/meson.build | 1 + > drivers/net/nfp/nfp_cpp_bridge.c | 88 ++++++++++++++++++++++++++----- > drivers/net/nfp/nfp_cpp_bridge.h | 6 ++- > drivers/net/nfp/nfp_ethdev.c | 40 ++++++++++++-- > 6 files changed, 239 insertions(+), 19 deletions(-) > create mode 100644 drivers/net/nfp/flower/nfp_flower.c > create mode 100644 drivers/net/nfp/flower/nfp_flower.h > > diff --git a/drivers/net/nfp/flower/nfp_flower.c b/drivers/net/nfp/flower/nfp_flower.c > new file mode 100644 > index 0000000..1dddced > --- /dev/null > +++ b/drivers/net/nfp/flower/nfp_flower.c > @@ -0,0 +1,101 @@ > +/* SPDX-License-Identifier: BSD-3-Clause > + * Copyright(c) 2022 Corigine, Inc. > + * All rights reserved. > + */ > + > +#include > +#include > +#include > +#include > +#include > +#include > + > +#include "../nfp_common.h" > +#include "../nfp_logs.h" > +#include "../nfp_ctrl.h" > +#include "../nfp_cpp_bridge.h" > +#include "nfp_flower.h" > + > +static struct rte_service_spec flower_services[NFP_FLOWER_SERVICE_MAX] = { > +}; > + > +static int > +nfp_flower_enable_services(struct nfp_app_flower *app_flower) > +{ > + int i; > + int ret = 0; > + > + for (i = 0; i < NFP_FLOWER_SERVICE_MAX; i++) { > + /* Pass a pointer to the flower app to the service */ > + flower_services[i].callback_userdata = (void *)app_flower; > + > + /* Register the flower services */ > + ret = rte_service_component_register(&flower_services[i], > + &app_flower->flower_services_ids[i]); > + if (ret) { Compare vs 0 explicitly > + PMD_INIT_LOG(WARNING, > + "Could not register Flower PF vNIC service"); > + break; > + } > + > + PMD_INIT_LOG(INFO, "Flower PF vNIC service registered"); > + > + /* Map them to available service cores*/ > + ret = nfp_map_service(app_flower->flower_services_ids[i]); > + if (ret) Compare vs 0 explicitly > + break; > + } > + > + return ret; > +} > + > +int > +nfp_init_app_flower(struct nfp_pf_dev *pf_dev) > +{ > + int ret; > + unsigned int numa_node; > + struct nfp_net_hw *pf_hw; > + struct nfp_app_flower *app_flower; > + > + numa_node = rte_socket_id(); > + > + /* Allocate memory for the Flower app */ > + app_flower = rte_zmalloc_socket("nfp_app_flower", sizeof(*app_flower), > + RTE_CACHE_LINE_SIZE, numa_node); > + if (app_flower == NULL) { > + ret = -ENOMEM; > + goto done; > + } > + > + pf_dev->app_priv = app_flower; > + > + /* Allocate memory for the PF AND ctrl vNIC here (hence the * 2) */ > + pf_hw = rte_zmalloc_socket("nfp_pf_vnic", 2 * sizeof(struct nfp_net_adapter), > + RTE_CACHE_LINE_SIZE, numa_node); > + if (pf_hw == NULL) { > + ret = -ENOMEM; > + goto app_cleanup; > + } > + > + /* Start up flower services */ > + if (nfp_flower_enable_services(app_flower)) { > + ret = -ESRCH; > + goto vnic_cleanup; > + } > + > + return 0; > + > +vnic_cleanup: > + rte_free(pf_hw); > +app_cleanup: > + rte_free(app_flower); > +done: > + return ret; > +} > + > +int > +nfp_secondary_init_app_flower(__rte_unused struct nfp_cpp *cpp) > +{ > + PMD_INIT_LOG(ERR, "Flower firmware not supported"); > + return -ENOTSUP; > +} > diff --git a/drivers/net/nfp/flower/nfp_flower.h b/drivers/net/nfp/flower/nfp_flower.h > new file mode 100644 > index 0000000..4a9b302 > --- /dev/null > +++ b/drivers/net/nfp/flower/nfp_flower.h > @@ -0,0 +1,22 @@ > +/* SPDX-License-Identifier: BSD-3-Clause > + * Copyright(c) 2022 Corigine, Inc. > + * All rights reserved. > + */ > + > +#ifndef _NFP_FLOWER_H_ > +#define _NFP_FLOWER_H_ > + > +enum nfp_flower_service { > + NFP_FLOWER_SERVICE_MAX > +}; > + > +/* The flower application's private structure */ > +struct nfp_app_flower { > + /* List of rte_service ID's for the flower app */ > + uint32_t flower_services_ids[NFP_FLOWER_SERVICE_MAX]; > +}; > + > +int nfp_init_app_flower(struct nfp_pf_dev *pf_dev); > +int nfp_secondary_init_app_flower(struct nfp_cpp *cpp); > + > +#endif /* _NFP_FLOWER_H_ */ > diff --git a/drivers/net/nfp/meson.build b/drivers/net/nfp/meson.build > index 810f02a..7ae3115 100644 > --- a/drivers/net/nfp/meson.build > +++ b/drivers/net/nfp/meson.build > @@ -6,6 +6,7 @@ if not is_linux or not dpdk_conf.get('RTE_ARCH_64') > reason = 'only supported on 64-bit Linux' > endif > sources = files( > + 'flower/nfp_flower.c', > 'nfpcore/nfp_cpp_pcie_ops.c', > 'nfpcore/nfp_nsp.c', > 'nfpcore/nfp_cppcore.c', > diff --git a/drivers/net/nfp/nfp_cpp_bridge.c b/drivers/net/nfp/nfp_cpp_bridge.c > index 0922ea9..9ac165a 100644 > --- a/drivers/net/nfp/nfp_cpp_bridge.c > +++ b/drivers/net/nfp/nfp_cpp_bridge.c > @@ -28,22 +28,86 @@ > static int nfp_cpp_bridge_serve_write(int sockfd, struct nfp_cpp *cpp); > static int nfp_cpp_bridge_serve_read(int sockfd, struct nfp_cpp *cpp); > static int nfp_cpp_bridge_serve_ioctl(int sockfd, struct nfp_cpp *cpp); > +static int nfp_cpp_bridge_service_func(void *args); > > -void nfp_register_cpp_service(struct nfp_cpp *cpp) > +static struct rte_service_spec cpp_service = { > + .name = "nfp_cpp_service", > + .callback = nfp_cpp_bridge_service_func, > +}; > + > +int > +nfp_map_service(uint32_t service_id) > { > - uint32_t *cpp_service_id = NULL; > - struct rte_service_spec service; > + int32_t ret; > + uint32_t slcore = 0; > + int32_t slcore_count; > + uint8_t service_count; > + const char *service_name; > + uint32_t slcore_array[RTE_MAX_LCORE]; > + uint8_t min_service_count = UINT8_MAX; > + > + slcore_count = rte_service_lcore_list(slcore_array, RTE_MAX_LCORE); > + if (slcore_count <= 0) { > + PMD_INIT_LOG(DEBUG, "No service cores found"); > + return -ENOENT; > + } > + > + /* > + * Find a service core with the least number of services already > + * registered to it > + */ > + while (slcore_count--) { > + service_count = rte_service_lcore_count_services(slcore_array[slcore_count]); > + if (service_count < min_service_count) { > + slcore = slcore_array[slcore_count]; > + min_service_count = service_count; > + } > + } > > - memset(&service, 0, sizeof(struct rte_service_spec)); > - snprintf(service.name, sizeof(service.name), "nfp_cpp_service"); > - service.callback = nfp_cpp_bridge_service_func; > - service.callback_userdata = (void *)cpp; > + service_name = rte_service_get_name(service_id); > + PMD_INIT_LOG(INFO, "Mapping service %s to core %u", service_name, slcore); > + ret = rte_service_map_lcore_set(service_id, slcore, 1); > + if (ret) { Compare vs 0 explicitly > + PMD_INIT_LOG(DEBUG, "Could not map flower service"); > + return -ENOENT; > + } > > - if (rte_service_component_register(&service, > - cpp_service_id)) > - RTE_LOG(WARNING, PMD, "NFP CPP bridge service register() failed"); > + rte_service_runstate_set(service_id, 1); > + rte_service_component_runstate_set(service_id, 1); > + rte_service_lcore_start(slcore); > + if (rte_service_may_be_active(slcore)) > + RTE_LOG(INFO, PMD, "The service %s is running", service_name); > else > - RTE_LOG(DEBUG, PMD, "NFP CPP bridge service registered"); > + RTE_LOG(INFO, PMD, "The service %s is not running", service_name); > + > + return 0; > +} > + > +int nfp_enable_cpp_service(struct nfp_cpp *cpp, enum nfp_app_id app_id) > +{ > + int ret = 0; > + uint32_t id = 0; > + > + cpp_service.callback_userdata = (void *)cpp; > + > + /* Register the cpp service */ > + ret = rte_service_component_register(&cpp_service, &id); > + if (ret) { Compare vs 0 explicitly > + PMD_INIT_LOG(WARNING, "Could not register nfp cpp service"); > + return -EINVAL; > + } > + > + PMD_INIT_LOG(INFO, "NFP cpp service registered"); > + > + /* Map it to available service core*/ > + ret = nfp_map_service(id); > + if (ret) { Compare vs 0 explicitly > + PMD_INIT_LOG(DEBUG, "Could not map nfp cpp service"); > + if (app_id == NFP_APP_FLOWER_NIC) > + return -EINVAL; > + } > + > + return 0; > } > > /* > @@ -307,7 +371,7 @@ void nfp_register_cpp_service(struct nfp_cpp *cpp) > * unaware of the CPP bridge performing the NFP kernel char driver for CPP > * accesses. > */ > -int32_t > +static int > nfp_cpp_bridge_service_func(void *args) > { > struct sockaddr address; > diff --git a/drivers/net/nfp/nfp_cpp_bridge.h b/drivers/net/nfp/nfp_cpp_bridge.h > index aea5fdc..dde50d7 100644 > --- a/drivers/net/nfp/nfp_cpp_bridge.h > +++ b/drivers/net/nfp/nfp_cpp_bridge.h > @@ -16,6 +16,8 @@ > #ifndef _NFP_CPP_BRIDGE_H_ > #define _NFP_CPP_BRIDGE_H_ > > +#include "nfp_common.h" > + > #define NFP_CPP_MEMIO_BOUNDARY (1 << 20) > #define NFP_BRIDGE_OP_READ 20 > #define NFP_BRIDGE_OP_WRITE 30 > @@ -24,8 +26,8 @@ > #define NFP_IOCTL 'n' > #define NFP_IOCTL_CPP_IDENTIFICATION _IOW(NFP_IOCTL, 0x8f, uint32_t) > > -void nfp_register_cpp_service(struct nfp_cpp *cpp); > -int32_t nfp_cpp_bridge_service_func(void *args); > +int nfp_map_service(uint32_t service_id); > +int nfp_enable_cpp_service(struct nfp_cpp *cpp, enum nfp_app_id app_id); > > #endif /* _NFP_CPP_BRIDGE_H_ */ > /* > diff --git a/drivers/net/nfp/nfp_ethdev.c b/drivers/net/nfp/nfp_ethdev.c > index 90dd01e..0b88749 100644 > --- a/drivers/net/nfp/nfp_ethdev.c > +++ b/drivers/net/nfp/nfp_ethdev.c > @@ -38,6 +38,8 @@ > #include "nfp_ctrl.h" > #include "nfp_cpp_bridge.h" > > +#include "flower/nfp_flower.h" > + > static int > nfp_net_pf_read_mac(struct nfp_app_nic *app_nic, int port) > { > @@ -837,7 +839,8 @@ > } > > static int > -nfp_pf_init(struct rte_pci_device *pci_dev) > +nfp_pf_init(struct rte_pci_device *pci_dev, > + struct rte_pci_driver *pci_drv) > { > int ret; > int err = 0; > @@ -964,6 +967,16 @@ > goto hwqueues_cleanup; > } > break; > + case NFP_APP_FLOWER_NIC: > + PMD_INIT_LOG(INFO, "Initializing Flower"); > + pci_dev->device.driver = &pci_drv->driver; > + ret = nfp_init_app_flower(pf_dev); > + if (ret) { Compare vs 0 explicitly > + PMD_INIT_LOG(ERR, "Could not initialize Flower!"); > + pci_dev->device.driver = NULL; > + goto hwqueues_cleanup; > + } > + break; > default: > PMD_INIT_LOG(ERR, "Unsupported Firmware loaded"); > ret = -EINVAL; > @@ -971,7 +984,12 @@ > } > > /* register the CPP bridge service here for primary use */ > - nfp_register_cpp_service(pf_dev->cpp); > + ret = nfp_enable_cpp_service(pf_dev->cpp, pf_dev->app_id); > + if (ret) { Compare vs 0 explicitly > + PMD_INIT_LOG(ERR, "Enable cpp service failed."); > + ret = -EINVAL; > + goto hwqueues_cleanup; > + } > > return 0; > > @@ -1096,6 +1114,14 @@ > goto sym_tbl_cleanup; > } > break; > + case NFP_APP_FLOWER_NIC: > + PMD_INIT_LOG(INFO, "Initializing Flower"); > + ret = nfp_secondary_init_app_flower(cpp); > + if (ret) { Compare vs 0 explicitly > + PMD_INIT_LOG(ERR, "Could not initialize Flower!"); > + goto sym_tbl_cleanup; > + } > + break; > default: > PMD_INIT_LOG(ERR, "Unsupported Firmware loaded"); > ret = -EINVAL; > @@ -1106,7 +1132,11 @@ > goto sym_tbl_cleanup; > > /* Register the CPP bridge service for the secondary too */ > - nfp_register_cpp_service(cpp); > + ret = nfp_enable_cpp_service(cpp, app_id); > + if (ret) { Compare vs 0 explicitly > + PMD_INIT_LOG(ERR, "Enable cpp service failed."); > + ret = -EINVAL; > + } > > sym_tbl_cleanup: > free(sym_tbl); > @@ -1115,11 +1145,11 @@ > } > > static int > -nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, > +nfp_pf_pci_probe(struct rte_pci_driver *pci_drv, > struct rte_pci_device *dev) > { > if (rte_eal_process_type() == RTE_PROC_PRIMARY) > - return nfp_pf_init(dev); > + return nfp_pf_init(dev, pci_drv); > else > return nfp_pf_secondary_init(dev); > }