Re: [dpdk-users] RSS Hash not working for XL710/X710 NICs for some RX mbuf sizes

Re: [dpdk-users] RSS Hash not working for XL710/X710 NICs for some RX mbuf sizes

[megha syam]

Hi Dumitru,
I am facing a similar problem with XL710 cards in my setup.
Is there a fix you have identified for this problem?
Thanks,
Syam

[dpdk-users] RSS Hash not working for XL710/X710 NICs for some RX mbuf sizes

[Take Ceara]

Hi,

Is there any known issue regarding the i40e DPDK driver when having
RSS hashing enabled in DPDK 16.04?
I've noticed that for some specific receive mbuf sizes the RSS hash is
always set to 0 for incoming packets.

I have a setup with two XL710 ports connected back to back. The simple
test program below sends fixed TCP packets from port 0 to port 1. The
L5 payload is added in the packet in such a way that the packet
consumes exactly one TX mbuf. For some values of the RX mbuf size the
incoming mbuf has the hash.rss == 0 even though the PKT_RX_RSS_HASH
flag is set in ol_flags. In my code the TX/RX mbuf sizes are
controlled by the RX_MBUF_SIZE and TX_MBUF_SIZE macros.

As an example, with some of the following TX/RX sizes the assert that
checks if the RSS hash is non-zero fails and with the other it passes:

RX_MBUF_SIZE  TX_MBUF_SIZE assert
=================================
1024          1024         fail
1025          1024         ok
1024          2048         fail
2048          2048         fail
2048          2047         fail
2049          2048         ok

On the same setup I have another loopback connection between two
82599ES 10G NICs and when I run exactly the same test the RSS hash is
always correct in all cases.

$ $RTE_SDK/tools/dpdk_nic_bind.py -s

Network devices using DPDK-compatible driver
============================================
0000:02:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection'
drv=igb_uio unused=
0000:03:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection'
drv=igb_uio unused=
0000:82:00.0 'Ethernet Controller XL710 for 40GbE QSFP+' drv=igb_uio unused=
0000:83:00.0 'Ethernet Controller XL710 for 40GbE QSFP+' drv=igb_uio unused=

The command line I use for running the test on the 40G NICs is:

./build/test -c 0x1 -n 4 -m 1024 -w 0000:82:00.0 -w 0000:83:00.0

Thanks,
Dumitru Ceara

#include <stdbool.h>
#include <stdint.h>
#include <assert.h>
#include <unistd.h>

#include <rte_ethdev.h>
#include <rte_timer.h>
#include <rte_ip.h>
#include <rte_tcp.h>
#include <rte_udp.h>
#include <rte_errno.h>
#include <rte_arp.h>

#define MBUF_SIZE(frag_size) \
    ((frag_size) + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM)

#define RX_MBUF_SIZE MBUF_SIZE(RTE_MBUF_DEFAULT_DATAROOM)
#define TX_MBUF_SIZE MBUF_SIZE(RTE_MBUF_DEFAULT_DATAROOM)

#define MBUF_CACHE 512
#define MBUF_COUNT 1024

static struct rte_mempool *rx_mpool;
static struct rte_mempool *tx_mpool;

#define PORT_MAX_MTU 9198

#define L5_GET_LEN(pkt) (rte_pktmbuf_tailroom((pkt)))

#define PORT0  0
#define PORT1  1
#define QUEUE0 0
#define Q_CNT  1


struct rte_eth_conf default_port_config = {
    .rxmode = {
        .mq_mode        = ETH_MQ_RX_RSS,
        .max_rx_pkt_len = PORT_MAX_MTU,
        .split_hdr_size = 0,
        .header_split   = 0, /**< Header Split disabled */
        .hw_ip_checksum = 1, /**< IP checksum offload enabled */
        .hw_vlan_filter = 0, /**< VLAN filtering disabled */
        .jumbo_frame    = 1, /**< Jumbo Frame Support disabled */
        .hw_strip_crc   = 0, /**< CRC stripped by hardware */
    },
    .rx_adv_conf = {
        .rss_conf = {
            .rss_key = NULL,
            .rss_key_len = 0,
            .rss_hf = ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_TCP |
ETH_RSS_NONFRAG_IPV4_UDP,
        },
    },
    .txmode = {
        .mq_mode = ETH_MQ_TX_NONE,
    }
};

struct rte_eth_rxconf rx_conf = {
    .rx_thresh = {
        .pthresh = 8,
        .hthresh = 8,
        .wthresh = 4,
    },
    .rx_free_thresh = 64,
    .rx_drop_en = 0
};

struct rte_eth_txconf tx_conf = {
    .tx_thresh = {
        .pthresh = 36,
        .hthresh = 0,
        .wthresh = 0,
    },
    .tx_free_thresh = 64,
    .tx_rs_thresh = 32,
};

static void port_setup(uint32_t port)
{
    uint32_t queue;
    int ret;

    assert(rte_eth_dev_configure(port, Q_CNT, Q_CNT,
                                 &default_port_config) == 0);
    for (queue = 0; queue < Q_CNT; queue++) {
        ret = rte_eth_rx_queue_setup(port, queue, 128, SOCKET_ID_ANY,
                                     &rx_conf,
                                     rx_mpool);
        assert(ret == 0);
        ret = rte_eth_tx_queue_setup(port, queue, 128, SOCKET_ID_ANY,
                                     &tx_conf);
        assert(ret == 0);
    }

    assert(rte_eth_dev_start(port) == 0);
}

#define HDRS_SIZE                   \
        (sizeof(struct ether_hdr) + \
         sizeof(struct ipv4_hdr) +  \
         sizeof(struct tcp_hdr))

static struct rte_mbuf *get_tcp_pkt(uint16_t eth_port)
{
    struct rte_mbuf  *pkt;
    struct ether_hdr *eth_hdr;
    struct ipv4_hdr  *ip_hdr;
    struct tcp_hdr   *tcp_hdr;
    uint32_t          ip_hdr_len = sizeof(*ip_hdr);
    uint32_t          tcp_hdr_len = sizeof(*tcp_hdr);
    uint32_t          l5_len;

    assert(pkt = rte_pktmbuf_alloc(tx_mpool));

    pkt->port = eth_port;
    pkt->l2_len = sizeof(*eth_hdr);

    RTE_LOG(ERR, USER1, "1:head = %d, tail = %d, len = %d\n",
            rte_pktmbuf_headroom(pkt), rte_pktmbuf_tailroom(pkt),
            rte_pktmbuf_pkt_len(pkt));

    /* Reserve space for ETH + IP + TCP Headers.
     * Store how much tailroom we have.
     */
    eth_hdr = (struct ether_hdr *)rte_pktmbuf_append(pkt, HDRS_SIZE);
    assert(eth_hdr);
    l5_len = L5_GET_LEN(pkt);

    /* ETH Header. */
    rte_eth_macaddr_get(PORT0, &eth_hdr->s_addr);
    rte_eth_macaddr_get(PORT1, &eth_hdr->d_addr);
    eth_hdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4);

    /* IP Header. */
    ip_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
    ip_hdr->version_ihl = (4 << 4) | (ip_hdr_len >> 2);
    ip_hdr->type_of_service = 0;
    ip_hdr->total_length = rte_cpu_to_be_16(ip_hdr_len + tcp_hdr_len + l5_len);
    ip_hdr->packet_id = 0;
    ip_hdr->fragment_offset = rte_cpu_to_be_16(0);
    ip_hdr->time_to_live = 60;
    ip_hdr->next_proto_id = IPPROTO_TCP;
    ip_hdr->src_addr = rte_cpu_to_be_32(0x01010101);
    ip_hdr->dst_addr = rte_cpu_to_be_32(0x01010101);
    ip_hdr->hdr_checksum = rte_cpu_to_be_16(0);

    pkt->l3_len = ip_hdr_len;
    pkt->ol_flags |= PKT_TX_IP_CKSUM;

    /* TCP Header. */
    tcp_hdr = (struct tcp_hdr *)(ip_hdr + 1);
    tcp_hdr->src_port = rte_cpu_to_be_16(0x42);
    tcp_hdr->dst_port = rte_cpu_to_be_16(0x24);
    tcp_hdr->sent_seq = rte_cpu_to_be_32(0x1234);
    tcp_hdr->recv_ack = rte_cpu_to_be_32(0x1234);
    tcp_hdr->data_off = tcp_hdr_len >> 2 << 4;
    tcp_hdr->tcp_flags = TCP_FIN_FLAG;
    tcp_hdr->rx_win = rte_cpu_to_be_16(0xffff);
    tcp_hdr->tcp_urp = rte_cpu_to_be_16(0);

    pkt->ol_flags |= PKT_TX_TCP_CKSUM | PKT_TX_IPV4;
    pkt->l4_len = tcp_hdr_len;

    tcp_hdr->cksum = 0;
    tcp_hdr->cksum = rte_ipv4_phdr_cksum(ip_hdr, pkt->ol_flags);

    /* Add Payload. */
    assert(rte_pktmbuf_append(pkt, l5_len));

    RTE_LOG(ERR, USER1, "1:head = %d, tail = %d, len = %d\n",
            rte_pktmbuf_headroom(pkt), rte_pktmbuf_tailroom(pkt),
            rte_pktmbuf_pkt_len(pkt));

    return pkt;
}

int main(int argc, char **argv)
{
    struct rte_mbuf *tx_mbuf[3];

    rte_eal_init(argc, argv);

    rx_mpool = rte_mempool_create("rx_mpool", MBUF_COUNT, RX_MBUF_SIZE,
                                  0,
                                  sizeof(struct rte_pktmbuf_pool_private),
                                  rte_pktmbuf_pool_init, NULL,
                                  rte_pktmbuf_init, NULL,
                                  SOCKET_ID_ANY,
                                  0);

    tx_mpool = rte_mempool_create("tx_mpool", MBUF_COUNT, TX_MBUF_SIZE,
                                  0,
                                  sizeof(struct rte_pktmbuf_pool_private),
                                  rte_pktmbuf_pool_init, NULL,
                                  rte_pktmbuf_init, NULL,
                                  SOCKET_ID_ANY,
                                  0);

    assert(rx_mpool && tx_mpool);

    port_setup(PORT0);
    port_setup(PORT1);

    for (;;) {
        uint16_t no_rx_buffers;
        uint16_t i;
        struct rte_mbuf *rx_pkts[16];

        tx_mbuf[0] = get_tcp_pkt(PORT0);
        assert(rte_eth_tx_burst(PORT0, QUEUE0, tx_mbuf, 1) == 1);

        no_rx_buffers = rte_eth_rx_burst(PORT1, QUEUE0, rx_pkts, 16);
        for (i = 0; i < no_rx_buffers; i++) {
            RTE_LOG(ERR, USER1, "RX RSS HASH: %8lX %4X\n",
                    rx_pkts[i]->ol_flags,
                    rx_pkts[i]->hash.rss);

            assert(rx_pkts[i]->ol_flags == PKT_RX_RSS_HASH);
            assert(rx_pkts[i]->hash.rss != 0);

            rte_pktmbuf_free(rx_pkts[i]);
        }
    }

    return 0;
}