From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-io0-f171.google.com (mail-io0-f171.google.com [209.85.223.171]) by dpdk.org (Postfix) with ESMTP id DA9742BF4 for ; Mon, 18 Jul 2016 17:03:19 +0200 (CEST) Received: by mail-io0-f171.google.com with SMTP id q83so162206980iod.1 for ; Mon, 18 Jul 2016 08:03:19 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113; h=mime-version:from:date:message-id:subject:to:cc; bh=02JXzCMw4/6vLAUNAahRB7+zTpJbIf46SmhpIAb8UMw=; b=DQ+pPWLQlgpCo7LqKV7+M9WKNVl+rSa3Qm9RTwB8CjUZB2IzwpY7vTPH8MTDK8KfwB D2/Wgo3aMjcvQN/c4tN8WCnVqLDho4qJD5FLwUCT4BadYhuf+CDe0AeVCIng++0n6Nje d0Lr1vmPnhQaISA7mUje7s8AoxrV3Z8B93kfyjJJtYOgVd1NHIK8Htv5xTxVsmobHh7L tDBtgfPVNfpwl9aPWnujna8hnPDxYpQN6TvStwA7bVLqEbdL8vfrMWRyffxitCdpu6af Kvcn8KbeFYFtin4qAn/5e784HMu4Jb9I73FKYwrjl3vcZc3oYbBMDB6mhB7X3U0oZSNI 40Qw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20130820; h=x-gm-message-state:mime-version:from:date:message-id:subject:to:cc; bh=02JXzCMw4/6vLAUNAahRB7+zTpJbIf46SmhpIAb8UMw=; b=FYcNEVayYxSxVbliNI7yO7Qd48BecmtU1ve5uQEgLGs/OhLGdVluXnTQZQ0R+k+3lj d69YRDi9eB/59TPffOjTFJCG1m/bELG2AllYh4JiFYWVnVV1Bm9dAWEQoCtGyT1u6tFM 55s+YEPeLFLFxSMY/uyIAC8U2u5hG5Abc6zBsvF4+sbPTFk9P6myA/QAfXTF4gxM2Xlj vVwrTjqs2pFNxB0xkhJXvdqaAz8T52xdip7M94lCUSh3KuZtdE3mz58VXlCUjqQ0ASqz peGWFxMgxK7EV7TU49tozBJMRDMwH0m02scCUShPXT9vWe40l63BN+k9PpSvJaPgVMK9 p+/w== X-Gm-Message-State: ALyK8tJK3f+fHbHXr5ZCmImTlvfMMLAmPg79HlMDySTRyT0GxXN8aJXU9MsvRA9H5f3yxsd7LlkmXbyOb4HrIw== X-Received: by 10.107.192.69 with SMTP id q66mr657708iof.5.1468854199009; Mon, 18 Jul 2016 08:03:19 -0700 (PDT) MIME-Version: 1.0 Received: by 10.107.7.82 with HTTP; Mon, 18 Jul 2016 08:02:58 -0700 (PDT) From: Take Ceara Date: Mon, 18 Jul 2016 17:02:58 +0200 Message-ID: To: users@dpdk.org Cc: helin.zhang@intel.com, jingjing.wu@intel.com Content-Type: text/plain; charset=UTF-8 Subject: [dpdk-users] RSS Hash not working for XL710/X710 NICs for some RX mbuf sizes X-BeenThere: users@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: usage discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 18 Jul 2016 15:03:20 -0000 Hi, Is there any known issue regarding the i40e DPDK driver when having RSS hashing enabled in DPDK 16.04? I've noticed that for some specific receive mbuf sizes the RSS hash is always set to 0 for incoming packets. I have a setup with two XL710 ports connected back to back. The simple test program below sends fixed TCP packets from port 0 to port 1. The L5 payload is added in the packet in such a way that the packet consumes exactly one TX mbuf. For some values of the RX mbuf size the incoming mbuf has the hash.rss == 0 even though the PKT_RX_RSS_HASH flag is set in ol_flags. In my code the TX/RX mbuf sizes are controlled by the RX_MBUF_SIZE and TX_MBUF_SIZE macros. As an example, with some of the following TX/RX sizes the assert that checks if the RSS hash is non-zero fails and with the other it passes: RX_MBUF_SIZE TX_MBUF_SIZE assert ================================= 1024 1024 fail 1025 1024 ok 1024 2048 fail 2048 2048 fail 2048 2047 fail 2049 2048 ok On the same setup I have another loopback connection between two 82599ES 10G NICs and when I run exactly the same test the RSS hash is always correct in all cases. $ $RTE_SDK/tools/dpdk_nic_bind.py -s Network devices using DPDK-compatible driver ============================================ 0000:02:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused= 0000:03:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused= 0000:82:00.0 'Ethernet Controller XL710 for 40GbE QSFP+' drv=igb_uio unused= 0000:83:00.0 'Ethernet Controller XL710 for 40GbE QSFP+' drv=igb_uio unused= The command line I use for running the test on the 40G NICs is: ./build/test -c 0x1 -n 4 -m 1024 -w 0000:82:00.0 -w 0000:83:00.0 Thanks, Dumitru Ceara #include #include #include #include #include #include #include #include #include #include #include #define MBUF_SIZE(frag_size) \ ((frag_size) + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM) #define RX_MBUF_SIZE MBUF_SIZE(RTE_MBUF_DEFAULT_DATAROOM) #define TX_MBUF_SIZE MBUF_SIZE(RTE_MBUF_DEFAULT_DATAROOM) #define MBUF_CACHE 512 #define MBUF_COUNT 1024 static struct rte_mempool *rx_mpool; static struct rte_mempool *tx_mpool; #define PORT_MAX_MTU 9198 #define L5_GET_LEN(pkt) (rte_pktmbuf_tailroom((pkt))) #define PORT0 0 #define PORT1 1 #define QUEUE0 0 #define Q_CNT 1 struct rte_eth_conf default_port_config = { .rxmode = { .mq_mode = ETH_MQ_RX_RSS, .max_rx_pkt_len = PORT_MAX_MTU, .split_hdr_size = 0, .header_split = 0, /**< Header Split disabled */ .hw_ip_checksum = 1, /**< IP checksum offload enabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 1, /**< Jumbo Frame Support disabled */ .hw_strip_crc = 0, /**< CRC stripped by hardware */ }, .rx_adv_conf = { .rss_conf = { .rss_key = NULL, .rss_key_len = 0, .rss_hf = ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV4_UDP, }, }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, } }; struct rte_eth_rxconf rx_conf = { .rx_thresh = { .pthresh = 8, .hthresh = 8, .wthresh = 4, }, .rx_free_thresh = 64, .rx_drop_en = 0 }; struct rte_eth_txconf tx_conf = { .tx_thresh = { .pthresh = 36, .hthresh = 0, .wthresh = 0, }, .tx_free_thresh = 64, .tx_rs_thresh = 32, }; static void port_setup(uint32_t port) { uint32_t queue; int ret; assert(rte_eth_dev_configure(port, Q_CNT, Q_CNT, &default_port_config) == 0); for (queue = 0; queue < Q_CNT; queue++) { ret = rte_eth_rx_queue_setup(port, queue, 128, SOCKET_ID_ANY, &rx_conf, rx_mpool); assert(ret == 0); ret = rte_eth_tx_queue_setup(port, queue, 128, SOCKET_ID_ANY, &tx_conf); assert(ret == 0); } assert(rte_eth_dev_start(port) == 0); } #define HDRS_SIZE \ (sizeof(struct ether_hdr) + \ sizeof(struct ipv4_hdr) + \ sizeof(struct tcp_hdr)) static struct rte_mbuf *get_tcp_pkt(uint16_t eth_port) { struct rte_mbuf *pkt; struct ether_hdr *eth_hdr; struct ipv4_hdr *ip_hdr; struct tcp_hdr *tcp_hdr; uint32_t ip_hdr_len = sizeof(*ip_hdr); uint32_t tcp_hdr_len = sizeof(*tcp_hdr); uint32_t l5_len; assert(pkt = rte_pktmbuf_alloc(tx_mpool)); pkt->port = eth_port; pkt->l2_len = sizeof(*eth_hdr); RTE_LOG(ERR, USER1, "1:head = %d, tail = %d, len = %d\n", rte_pktmbuf_headroom(pkt), rte_pktmbuf_tailroom(pkt), rte_pktmbuf_pkt_len(pkt)); /* Reserve space for ETH + IP + TCP Headers. * Store how much tailroom we have. */ eth_hdr = (struct ether_hdr *)rte_pktmbuf_append(pkt, HDRS_SIZE); assert(eth_hdr); l5_len = L5_GET_LEN(pkt); /* ETH Header. */ rte_eth_macaddr_get(PORT0, ð_hdr->s_addr); rte_eth_macaddr_get(PORT1, ð_hdr->d_addr); eth_hdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4); /* IP Header. */ ip_hdr = (struct ipv4_hdr *)(eth_hdr + 1); ip_hdr->version_ihl = (4 << 4) | (ip_hdr_len >> 2); ip_hdr->type_of_service = 0; ip_hdr->total_length = rte_cpu_to_be_16(ip_hdr_len + tcp_hdr_len + l5_len); ip_hdr->packet_id = 0; ip_hdr->fragment_offset = rte_cpu_to_be_16(0); ip_hdr->time_to_live = 60; ip_hdr->next_proto_id = IPPROTO_TCP; ip_hdr->src_addr = rte_cpu_to_be_32(0x01010101); ip_hdr->dst_addr = rte_cpu_to_be_32(0x01010101); ip_hdr->hdr_checksum = rte_cpu_to_be_16(0); pkt->l3_len = ip_hdr_len; pkt->ol_flags |= PKT_TX_IP_CKSUM; /* TCP Header. */ tcp_hdr = (struct tcp_hdr *)(ip_hdr + 1); tcp_hdr->src_port = rte_cpu_to_be_16(0x42); tcp_hdr->dst_port = rte_cpu_to_be_16(0x24); tcp_hdr->sent_seq = rte_cpu_to_be_32(0x1234); tcp_hdr->recv_ack = rte_cpu_to_be_32(0x1234); tcp_hdr->data_off = tcp_hdr_len >> 2 << 4; tcp_hdr->tcp_flags = TCP_FIN_FLAG; tcp_hdr->rx_win = rte_cpu_to_be_16(0xffff); tcp_hdr->tcp_urp = rte_cpu_to_be_16(0); pkt->ol_flags |= PKT_TX_TCP_CKSUM | PKT_TX_IPV4; pkt->l4_len = tcp_hdr_len; tcp_hdr->cksum = 0; tcp_hdr->cksum = rte_ipv4_phdr_cksum(ip_hdr, pkt->ol_flags); /* Add Payload. */ assert(rte_pktmbuf_append(pkt, l5_len)); RTE_LOG(ERR, USER1, "1:head = %d, tail = %d, len = %d\n", rte_pktmbuf_headroom(pkt), rte_pktmbuf_tailroom(pkt), rte_pktmbuf_pkt_len(pkt)); return pkt; } int main(int argc, char **argv) { struct rte_mbuf *tx_mbuf[3]; rte_eal_init(argc, argv); rx_mpool = rte_mempool_create("rx_mpool", MBUF_COUNT, RX_MBUF_SIZE, 0, sizeof(struct rte_pktmbuf_pool_private), rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, NULL, SOCKET_ID_ANY, 0); tx_mpool = rte_mempool_create("tx_mpool", MBUF_COUNT, TX_MBUF_SIZE, 0, sizeof(struct rte_pktmbuf_pool_private), rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, NULL, SOCKET_ID_ANY, 0); assert(rx_mpool && tx_mpool); port_setup(PORT0); port_setup(PORT1); for (;;) { uint16_t no_rx_buffers; uint16_t i; struct rte_mbuf *rx_pkts[16]; tx_mbuf[0] = get_tcp_pkt(PORT0); assert(rte_eth_tx_burst(PORT0, QUEUE0, tx_mbuf, 1) == 1); no_rx_buffers = rte_eth_rx_burst(PORT1, QUEUE0, rx_pkts, 16); for (i = 0; i < no_rx_buffers; i++) { RTE_LOG(ERR, USER1, "RX RSS HASH: %8lX %4X\n", rx_pkts[i]->ol_flags, rx_pkts[i]->hash.rss); assert(rx_pkts[i]->ol_flags == PKT_RX_RSS_HASH); assert(rx_pkts[i]->hash.rss != 0); rte_pktmbuf_free(rx_pkts[i]); } } return 0; }