From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id A0117A04BC; Thu, 8 Oct 2020 10:53:25 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 53AED1BC74; Thu, 8 Oct 2020 10:52:18 +0200 (CEST) Received: from git-send-mailer.rdmz.labs.mlnx (unknown [37.142.13.130]) by dpdk.org (Postfix) with ESMTP id 90F701BC1A for ; Thu, 8 Oct 2020 10:52:11 +0200 (CEST) From: Bing Zhao To: thomas@monjalon.net, orika@nvidia.com, ferruh.yigit@intel.com, arybchenko@solarflare.com, mdr@ashroe.eu, nhorman@tuxdriver.com, bernard.iremonger@intel.com, beilei.xing@intel.com, wenzhuo.lu@intel.com Cc: dev@dpdk.org Date: Thu, 8 Oct 2020 16:51:37 +0800 Message-Id: <1602147098-9768-6-git-send-email-bingz@nvidia.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1602147098-9768-1-git-send-email-bingz@nvidia.com> References: <1601511962-21532-1-git-send-email-bingz@nvidia.com> <1602147098-9768-1-git-send-email-bingz@nvidia.com> Subject: [dpdk-dev] [PATCH v2 5/6] app/testpmd: change hairpin queues setup X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" A new parameter `hairpin-mode` is introduced to the testpmd command line. Bitmask value is used to provide more flexible configuration. This parameter should be used when `hairpinq` is specified in the command line. Bit 0 in the LSB indicates the hairpin will use the loop mode. The previous port RX queue will be connected to the current port TX queue. Bit 1 in the LSB indicates the hairpin will use pair port mode. The even index port will be paired with the next odd index port. If the total number of probed port is odd, then the last one will be paired to itself. If this byte is zero, then each port will be paired to itself. Bit 0 takes a higher priority in the checking. Bit 4 in the second bytes indicate if the hairpin will use explicit TX flow mode. e.g. in the command line, "--hairpinq=2 --hairpin-mode=0x11" If not set, default value zero will be used and the behavior will try to get align with the previous single port mode. If the ports belong to different vendors' NICs, it is suggested to use the `self` hairpin mode only. Since hairpin configures the hardware resources, the port mask of packets forwarding engine will not be used here. Signed-off-by: Bing Zhao --- v2: move the hairpin bind/unbind into start/stop to support hot-plug and hot-unplug --- app/test-pmd/parameters.c | 15 ++++++ app/test-pmd/testpmd.c | 125 ++++++++++++++++++++++++++++++++++++++++++++-- app/test-pmd/testpmd.h | 2 + 3 files changed, 138 insertions(+), 4 deletions(-) diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c index 1ead595..991029d 100644 --- a/app/test-pmd/parameters.c +++ b/app/test-pmd/parameters.c @@ -221,6 +221,9 @@ "enabled\n"); printf(" --record-core-cycles: enable measurement of CPU cycles.\n"); printf(" --record-burst-stats: enable display of RX and TX bursts.\n"); + printf(" --hairpin-mode=0xXX: bitmask set the hairpin port mode.\n " + " 0x10 - explicit tx rule, 0x02 - hairpin ports paired\n" + " 0x01 - hairpin ports loop, 0x00 - hairpin port self\n"); } #ifdef RTE_LIBRTE_CMDLINE @@ -644,6 +647,7 @@ { "rxd", 1, 0, 0 }, { "txd", 1, 0, 0 }, { "hairpinq", 1, 0, 0 }, + { "hairpin-mode", 1, 0, 0 }, { "burst", 1, 0, 0 }, { "mbcache", 1, 0, 0 }, { "txpt", 1, 0, 0 }, @@ -1111,6 +1115,17 @@ rte_exit(EXIT_FAILURE, "Either rx or tx queues should " "be non-zero\n"); } + if (!strcmp(lgopts[opt_idx].name, "hairpin-mode")) { + char *end = NULL; + unsigned int n; + + errno = 0; + n = strtoul(optarg, &end, 0); + if (errno != 0 || end == optarg) + rte_exit(EXIT_FAILURE, "hairpin mode invalid\n"); + else + hairpin_mode = (uint16_t)n; + } if (!strcmp(lgopts[opt_idx].name, "burst")) { n = atoi(optarg); if (n == 0) { diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c index ccba71c..344de83 100644 --- a/app/test-pmd/testpmd.c +++ b/app/test-pmd/testpmd.c @@ -367,6 +367,9 @@ struct fwd_engine * fwd_engines[] = { /* Clear ptypes on port initialization. */ uint8_t clear_ptypes = true; +/* Hairpin ports configuration mode. */ +uint16_t hairpin_mode; + /* Pretty printing of ethdev events */ static const char * const eth_event_desc[] = { [RTE_ETH_EVENT_UNKNOWN] = "unknown", @@ -2345,7 +2348,7 @@ struct extmem_param { /* Configure the Rx and Tx hairpin queues for the selected port. */ static int -setup_hairpin_queues(portid_t pi) +setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) { queueid_t qi; struct rte_eth_hairpin_conf hairpin_conf = { @@ -2354,10 +2357,49 @@ struct extmem_param { int i; int diag; struct rte_port *port = &ports[pi]; + uint16_t peer_rx_port = pi; + uint16_t peer_tx_port = pi; + uint32_t manual = 1; + uint32_t tx_exp = hairpin_mode & 0x10; + + if (!(hairpin_mode & 0xf)) { + peer_rx_port = pi; + peer_tx_port = pi; + manual = 0; + } else if (hairpin_mode & 0x1) { + peer_tx_port = rte_eth_find_next_owned_by(pi + 1, + RTE_ETH_DEV_NO_OWNER); + if (peer_tx_port >= RTE_MAX_ETHPORTS) + peer_tx_port = rte_eth_find_next_owned_by(0, + RTE_ETH_DEV_NO_OWNER); + if (p_pi != RTE_MAX_ETHPORTS) { + peer_rx_port = p_pi; + } else { + uint16_t next_pi; + + /* Last port will be the peer RX port of the first. */ + RTE_ETH_FOREACH_DEV(next_pi) + peer_rx_port = next_pi; + } + manual = 1; + } else if (hairpin_mode & 0x2) { + if (cnt_pi & 0x1) { + peer_rx_port = p_pi; + } else { + peer_rx_port = rte_eth_find_next_owned_by(pi + 1, + RTE_ETH_DEV_NO_OWNER); + if (peer_rx_port >= RTE_MAX_ETHPORTS) + peer_rx_port = pi; + } + peer_tx_port = peer_rx_port; + manual = 1; + } for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { - hairpin_conf.peers[0].port = pi; + hairpin_conf.peers[0].port = peer_rx_port; hairpin_conf.peers[0].queue = i + nb_rxq; + hairpin_conf.manual_bind = !!manual; + hairpin_conf.tx_explicit = !!tx_exp; diag = rte_eth_tx_hairpin_queue_setup (pi, qi, nb_txd, &hairpin_conf); i++; @@ -2377,8 +2419,10 @@ struct extmem_param { return -1; } for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { - hairpin_conf.peers[0].port = pi; + hairpin_conf.peers[0].port = peer_tx_port; hairpin_conf.peers[0].queue = i + nb_txq; + hairpin_conf.manual_bind = !!manual; + hairpin_conf.tx_explicit = !!tx_exp; diag = rte_eth_rx_hairpin_queue_setup (pi, qi, nb_rxd, &hairpin_conf); i++; @@ -2405,6 +2449,12 @@ struct extmem_param { { int diag, need_check_link_status = -1; portid_t pi; + portid_t p_pi = RTE_MAX_ETHPORTS; + portid_t pl[RTE_MAX_ETHPORTS]; + portid_t peer_pl[RTE_MAX_ETHPORTS]; + uint16_t cnt_pi = 0; + uint16_t cfg_pi = 0; + int peer_pi; queueid_t qi; struct rte_port *port; struct rte_ether_addr mac_addr; @@ -2544,7 +2594,7 @@ struct extmem_param { return -1; } /* setup hairpin queues */ - if (setup_hairpin_queues(pi) != 0) + if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0) return -1; } configure_rxtx_dump_callbacks(verbose_level); @@ -2557,6 +2607,9 @@ struct extmem_param { pi); } + p_pi = pi; + cnt_pi++; + /* start port */ if (rte_eth_dev_start(pi) < 0) { printf("Fail to start port %d\n", pi); @@ -2581,6 +2634,8 @@ struct extmem_param { /* at least one port started, need checking link status */ need_check_link_status = 1; + + pl[cfg_pi++] = pi; } if (need_check_link_status == 1 && !no_link_check) @@ -2588,6 +2643,50 @@ struct extmem_param { else if (need_check_link_status == 0) printf("Please stop the ports first\n"); + if (hairpin_mode & 0xf) { + uint16_t i; + int j; + + /* bind all started hairpin ports */ + for (i = 0; i < cfg_pi; i++) { + pi = pl[i]; + /* bind current TX to all peer RX */ + peer_pi = rte_eth_hairpin_get_peer_ports(pi, + peer_pl, 1); + if (peer_pi < 0) + return peer_pi; + for (j = 0; j < peer_pi; j++) { + if (!port_is_started(peer_pl[j])) + continue; + diag = rte_eth_hairpin_bind(pi, peer_pl[j]); + if (diag < 0) { + printf("Error during binding " + "hairpin tx port %u to %u: %s", + pi, peer_pl[j], + rte_strerror(-diag)); + return -1; + } + } + /* bind all peer TX to current RX */ + peer_pi = rte_eth_hairpin_get_peer_ports(pi, + peer_pl, 0); + if (peer_pi < 0) + return peer_pi; + for (j = 0; j < peer_pi; j++) { + if (!port_is_started(peer_pl[j])) + continue; + diag = rte_eth_hairpin_bind(peer_pl[j], pi); + if (diag < 0) { + printf("Error during binding " + "hairpin tx port %u to %u: %s", + peer_pl[j], pi, + rte_strerror(-diag)); + return -1; + } + } + } + } + printf("Done\n"); return 0; } @@ -2598,6 +2697,8 @@ struct extmem_param { portid_t pi; struct rte_port *port; int need_check_link_status = 0; + portid_t peer_pl[RTE_MAX_ETHPORTS]; + int peer_pi; if (dcb_test) { dcb_test = 0; @@ -2628,6 +2729,22 @@ struct extmem_param { RTE_PORT_HANDLING) == 0) continue; + if (hairpin_mode & 0xf) { + int j; + + rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS); + /* unbind all peer TX from current RX */ + peer_pi = rte_eth_hairpin_get_peer_ports(pi, + peer_pl, 0); + if (peer_pi < 0) + continue; + for (j = 0; j < peer_pi; j++) { + if (!port_is_started(peer_pl[j])) + continue; + rte_eth_hairpin_unbind(peer_pl[j], pi); + } + } + rte_eth_dev_stop(pi); if (rte_atomic16_cmpset(&(port->port_status), diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h index c7e7e41..29ede20 100644 --- a/app/test-pmd/testpmd.h +++ b/app/test-pmd/testpmd.h @@ -398,6 +398,8 @@ struct queue_stats_mappings { extern uint16_t stats_period; +extern uint16_t hairpin_mode; + #ifdef RTE_LIBRTE_LATENCY_STATS extern uint8_t latencystats_enabled; extern lcoreid_t latencystats_lcore_id; -- 1.8.3.1