From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from netronome.com (unknown [217.38.71.146]) by dpdk.org (Postfix) with ESMTP id 406FA5699 for ; Thu, 3 Jan 2019 09:56:22 +0100 (CET) Received: from netronome.com (localhost [127.0.0.1]) by netronome.com (8.15.2/8.15.2/Debian-10) with ESMTP id x038uMES023658 for ; Thu, 3 Jan 2019 08:56:22 GMT Received: (from root@localhost) by netronome.com (8.15.2/8.15.2/Submit) id x038uMee023657 for dev@dpdk.org; Thu, 3 Jan 2019 08:56:22 GMT From: Alejandro Lucero To: dev@dpdk.org Date: Thu, 3 Jan 2019 08:56:21 +0000 Message-Id: <20190103085621.23611-1-alejandro.lucero@netronome.com> X-Mailer: git-send-email 2.17.1 Subject: [dpdk-dev] [PATCH] net/nfp: add CPP bridge as service X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Jan 2019 08:56:22 -0000 The Netronome's Network Flow Processor chip is highly programmable with the goal of processing packets at high speed. Processing units and other chip components are available from the host through the PCIe CPP(Command Push Pull bus) interface. The NFP PF PMD configures a CPP handler for setting up and working with vNICs, perform actions like link up or down, or accessing extended stats from the MAC component. There exist NFP host tools which access the NFP components for programming and debugging but they require the CPP interface. When the PMD is bound to the PF, the DPDK app owns the CPP interface, so these host tools can not access the NFP through other means like NFP kernel drivers. This patch adds a CPP bridge using the rte_service API which can be enabled by a DPDK app. Interestingly, DPDK clients like OVS will not enable specific service cores, but this can be performed with a secondary process specifically enabling this CPP bridge service and therefore giving access to the NFP to those host tools. Signed-off-by: Alejandro Lucero --- drivers/net/nfp/nfp_net.c | 393 +++++++++++++++++++++++++++++++++- drivers/net/nfp/nfp_net_pmd.h | 1 + 2 files changed, 393 insertions(+), 1 deletion(-) diff --git a/drivers/net/nfp/nfp_net.c b/drivers/net/nfp/nfp_net.c index 05a44a2a9..881cbd504 100644 --- a/drivers/net/nfp/nfp_net.c +++ b/drivers/net/nfp/nfp_net.c @@ -54,6 +54,7 @@ #include #include #include +#include #include "nfpcore/nfp_cpp.h" #include "nfpcore/nfp_nffw.h" @@ -66,6 +67,16 @@ #include "nfp_net_logs.h" #include "nfp_net_ctrl.h" +#include +#include +#include +#include +#include +#include +#include + +#undef CPP_BRIDGE_DEBUG + /* Prototypes */ static void nfp_net_close(struct rte_eth_dev *dev); static int nfp_net_configure(struct rte_eth_dev *dev); @@ -2949,14 +2960,371 @@ nfp_net_init(struct rte_eth_dev *eth_dev) return err; } +#define NFP_CPP_MEMIO_BOUNDARY (1 << 20) + +/* + * Serving a write request to NFP from host programs. The request + * sends the write size and the CPP target. The bridge makes use + * of CPP interface handler configured by the PMD setup. + */ +static int +nfp_cpp_bridge_serve_write(int sockfd, struct nfp_cpp *cpp) +{ + struct nfp_cpp_area *area; + off_t offset, nfp_offset; + uint32_t cpp_id, pos, len; + uint32_t tmpbuf[16]; + size_t count, curlen, totlen = 0; + int err = 0; + +#ifdef CPP_BRIDGE_DEBUG + printf("%s: offset size %lu, count_size: %lu\n", __func__, + sizeof(off_t), sizeof(size_t)); +#endif + + /* Reading the count param */ + err = recv(sockfd, &count, sizeof(off_t), 0); + if (err != sizeof(off_t)) + return -EINVAL; + + curlen = count; + + /* Reading the offset param */ + err = recv(sockfd, &offset, sizeof(off_t), 0); + if (err != sizeof(off_t)) + return -EINVAL; + + /* Obtain target's CPP ID and offset in target */ + cpp_id = (offset >> 40) << 8; + nfp_offset = offset & ((1ull << 40) - 1); + +#ifdef CPP_BRIDGE_DEBUG + printf("%s: count %lu and offset %ld\n", __func__, count, + offset); + printf("%s: cpp_id %08x and nfp_offset %ld\n", __func__, + cpp_id, nfp_offset); +#endif + + /* Adjust length if not aligned */ + if (((nfp_offset + (off_t)count - 1) & ~(NFP_CPP_MEMIO_BOUNDARY - 1)) != + (nfp_offset & ~(NFP_CPP_MEMIO_BOUNDARY - 1))) { + curlen = NFP_CPP_MEMIO_BOUNDARY - + (nfp_offset & (NFP_CPP_MEMIO_BOUNDARY - 1)); + } + + while (count > 0) { + /* configure a CPP PCIe2CPP BAR for mapping the CPP target */ + area = nfp_cpp_area_alloc_with_name(cpp, cpp_id, "nfp.cdev", + nfp_offset, curlen); + if (!area) { + printf("%s: area alloc fail\n", __func__); + return -EIO; + } + + /* mapping the target */ + err = nfp_cpp_area_acquire(area); + if (err < 0) { + printf("%s: area acquire failed\n", __func__); + nfp_cpp_area_free(area); + return -EIO; + } + + for (pos = 0; pos < curlen; pos += len) { + len = curlen - pos; + if (len > sizeof(tmpbuf)) + len = sizeof(tmpbuf); + +#ifdef CPP_BRIDGE_DEBUG + printf("%s: Receiving %u of %lu\n", __func__, len, + count); +#endif + err = recv(sockfd, tmpbuf, len, MSG_WAITALL); + if (err != (int)len) { + printf("%s: error when receiving, %d of %lu\n", + __func__, err, count); + nfp_cpp_area_release(area); + nfp_cpp_area_free(area); + return -EIO; + } + err = nfp_cpp_area_write(area, pos, tmpbuf, len); + if (err < 0) { + printf("%s: nfp_cpp_area_write error\n", + __func__); + nfp_cpp_area_release(area); + nfp_cpp_area_free(area); + return -EIO; + } + } + + nfp_offset += pos; + totlen += pos; + nfp_cpp_area_release(area); + nfp_cpp_area_free(area); + + count -= pos; + curlen = (count > NFP_CPP_MEMIO_BOUNDARY) ? + NFP_CPP_MEMIO_BOUNDARY : count; + } + + return 0; +} + +/* + * Serving a read request to NFP from host programs. The request + * sends the read size and the CPP target. The bridge makes use + * of CPP interface handler configured by the PMD setup. The read + * data is sent to the requester using the same socket. + */ +static int +nfp_cpp_bridge_serve_read(int sockfd, struct nfp_cpp *cpp) +{ + struct nfp_cpp_area *area; + off_t offset, nfp_offset; + uint32_t cpp_id, pos, len; + uint32_t tmpbuf[16]; + size_t count, curlen, totlen = 0; + int err = 0; + +#ifdef CPP_BRIDGE_DEBUG + printf("%s: offset size %lu, count_size: %lu\n", __func__, + sizeof(off_t), sizeof(size_t)); +#endif + + /* Reading the count param */ + err = recv(sockfd, &count, sizeof(off_t), 0); + if (err != sizeof(off_t)) + return -EINVAL; + + curlen = count; + + /* Reading the offset param */ + err = recv(sockfd, &offset, sizeof(off_t), 0); + if (err != sizeof(off_t)) + return -EINVAL; + + /* Obtain target's CPP ID and offset in target */ + cpp_id = (offset >> 40) << 8; + nfp_offset = offset & ((1ull << 40) - 1); + +#ifdef CPP_BRIDGE_DEBUG + printf("%s: count %lu and offset %ld\n", __func__, count, offset); + printf("%s: cpp_id %08x and nfp_offset %ld\n", __func__, cpp_id, + nfp_offset); +#endif + + /* Adjust length if not aligned */ + if (((nfp_offset + (off_t)count - 1) & ~(NFP_CPP_MEMIO_BOUNDARY - 1)) != + (nfp_offset & ~(NFP_CPP_MEMIO_BOUNDARY - 1))) { + curlen = NFP_CPP_MEMIO_BOUNDARY - + (nfp_offset & (NFP_CPP_MEMIO_BOUNDARY - 1)); + } + + while (count > 0) { + area = nfp_cpp_area_alloc_with_name(cpp, cpp_id, "nfp.cdev", + nfp_offset, curlen); + if (!area) { + printf("%s: area alloc failed\n", __func__); + return -EIO; + } + + err = nfp_cpp_area_acquire(area); + if (err < 0) { + printf("%s: area acquire failed\n", __func__); + nfp_cpp_area_free(area); + return -EIO; + } + + for (pos = 0; pos < curlen; pos += len) { + len = curlen - pos; + if (len > sizeof(tmpbuf)) + len = sizeof(tmpbuf); + + err = nfp_cpp_area_read(area, pos, tmpbuf, len); + if (err < 0) { + printf("%s: nfp_cpp_area_read error\n", + __func__); + nfp_cpp_area_release(area); + nfp_cpp_area_free(area); + return -EIO; + } +#ifdef CPP_BRIDGE_DEBUG + printf("%s: sending %u of %lu\n", __func__, len, count); +#endif + err = send(sockfd, tmpbuf, len, 0); + if (err != (int)len) { + printf("%s: error when sending: %d of %lu\n", + __func__, err, count); + nfp_cpp_area_release(area); + nfp_cpp_area_free(area); + return -EIO; + } + } + + nfp_offset += pos; + totlen += pos; + nfp_cpp_area_release(area); + nfp_cpp_area_free(area); + + count -= pos; + curlen = (count > NFP_CPP_MEMIO_BOUNDARY) ? + NFP_CPP_MEMIO_BOUNDARY : count; + } + return 0; +} + +#define NFP_IOCTL 'n' +#define NFP_IOCTL_CPP_IDENTIFICATION _IOW(NFP_IOCTL, 0x8f, uint32_t) +/* + * Serving a ioctl command from host NFP tools. This usually goes to + * a kernel driver char driver but it is not available when the PF is + * bound to the PMD. Currently just one ioctl command is served and it + * does not require any CPP access at all. + */ +static int +nfp_cpp_bridge_serve_ioctl(int sockfd, struct nfp_cpp *cpp) +{ + int cmd, err; + uint32_t ident_size, tmp; + + /* Reading now the IOCTL command */ + err = recv(sockfd, &cmd, 4, 0); + if (err != 4) { + printf("%s: read error from socket\n", __func__); + return -EIO; + } + + /* Only supporting NFP_IOCTL_CPP_IDENTIFICATION */ + if (cmd != NFP_IOCTL_CPP_IDENTIFICATION) { + printf("%s: unknown cmd %d\n", __func__, cmd); + return -EINVAL; + } + + err = recv(sockfd, &ident_size, 4, 0); + if (err != 4) { + printf("%s: read error from socket\n", __func__); + return -EIO; + } + + tmp = nfp_cpp_model(cpp); + +#ifdef CPP_BRIDGE_DEBUG + printf("%s: sending NFP model %08x\n", __func__, tmp); +#endif + err = send(sockfd, &tmp, 4, 0); + if (err != 4) { + printf("%s: error writing to socket\n", __func__); + return -EIO; + } + + tmp = cpp->interface; + +#ifdef CPP_BRIDGE_DEBUG + printf("%s: sending NFP interface %08x\n", __func__, tmp); +#endif + err = send(sockfd, &tmp, 4, 0); + if (err != 4) { + printf("%s: error writing to socket\n", __func__); + return -EIO; + } + + return 0; +} + +#define NFP_BRIDGE_OP_READ 20 +#define NFP_BRIDGE_OP_WRITE 30 +#define NFP_BRIDGE_OP_IOCTL 40 + +/* + * This is the code to be executed by a service core. The CPP bridge interface + * is based on a unix socket and requests usually received by a kernel char + * driver, read, write and ioctl, are handled by the CPP bridge. NFP host tools + * can be executed with a wrapper library and LD_LIBRARY being completely + * unaware of the CPP bridge performing the NFP kernel char driver for CPP + * accesses. + */ +static int32_t +nfp_cpp_bridge_service_func(void *args) +{ + struct sockaddr address; + struct nfp_cpp *cpp = args; + int sockfd, datafd, op, ret; + + unlink("/tmp/nfp_cpp"); + sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (sockfd < 0) { + printf("%s: socket creation error. Service failed\n", + __func__); + return -EIO; + } + + memset(&address, 0, sizeof(struct sockaddr)); + + address.sa_family = AF_UNIX; + strcpy(address.sa_data, "/tmp/nfp_cpp"); + + ret = bind(sockfd, (const struct sockaddr *)&address, + sizeof(struct sockaddr)); + if (ret < 0) { + printf("%s: bind error (%d). Service failed\n", __func__, + errno); + return ret; + } + + ret = listen(sockfd, 20); + if (ret < 0) { + printf("%s: listen error(%d). Service failed\n", __func__, + errno); + return ret; + } + + for (;;) { + datafd = accept(sockfd, NULL, NULL); + if (datafd < 0) { + printf("%s: accept call error (%d)\n", __func__, errno); + printf("%s: service failed\n", __func__); + return -EIO; + } + + while (1) { + ret = recv(datafd, &op, 4, 0); + if (ret <= 0) { +#ifdef CPP_BRIDGE_DEBUG + printf("%s: socket close\n", __func__); +#endif + break; + } + +#ifdef CPP_BRIDGE_DEBUG + printf("%s: getting op %u\n", __func__, op); +#endif + if (op == NFP_BRIDGE_OP_READ) + nfp_cpp_bridge_serve_read(datafd, cpp); + + if (op == NFP_BRIDGE_OP_WRITE) + nfp_cpp_bridge_serve_write(datafd, cpp); + + if (op == NFP_BRIDGE_OP_IOCTL) + nfp_cpp_bridge_serve_ioctl(datafd, cpp); + + if (op == 0) + break; + } + close(datafd); + } + close(sockfd); + + return 0; +} + static int nfp_pf_create_dev(struct rte_pci_device *dev, int port, int ports, struct nfp_cpp *cpp, struct nfp_hwinfo *hwinfo, int phys_port, struct nfp_rtsym_table *sym_tbl, void **priv) { struct rte_eth_dev *eth_dev; - struct nfp_net_hw *hw; + struct nfp_net_hw *hw = NULL; char *port_name; + struct rte_service_spec service; int retval; port_name = rte_zmalloc("nfp_pf_port_name", 100, 0); @@ -3027,6 +3395,29 @@ nfp_pf_create_dev(struct rte_pci_device *dev, int port, int ports, rte_free(port_name); + if (port == 0) { + /* + * The rte_service needs to be created just once per PMD. + * And the cpp handler needs to be linked to the service. + * Secondary processes will be used for debugging DPDK apps + * when requiring to use the CPP interface for accessing NFP + * components. And the cpp handler for secondary processes is + * available at this point. + */ + memset(&service, 0, sizeof(struct rte_service_spec)); + snprintf(service.name, sizeof(service.name), "nfp_cpp_service"); + service.callback = nfp_cpp_bridge_service_func; + service.callback_userdata = (void *)cpp; + + hw = (struct nfp_net_hw *)(eth_dev->data->dev_private); + + if (rte_service_component_register(&service, + &hw->nfp_cpp_service_id)) + printf("NFP CPP bridge service register() failed"); + else + printf("NFP CPP bridge service registered"); + } + return retval; probe_failed: diff --git a/drivers/net/nfp/nfp_net_pmd.h b/drivers/net/nfp/nfp_net_pmd.h index b01036df2..7913a29c2 100644 --- a/drivers/net/nfp/nfp_net_pmd.h +++ b/drivers/net/nfp/nfp_net_pmd.h @@ -456,6 +456,7 @@ struct nfp_net_hw { struct nfp_hwinfo *hwinfo; struct nfp_rtsym_table *sym_tbl; + uint32_t nfp_cpp_service_id; }; struct nfp_net_adapter { -- 2.17.1