Re: Can a DPDK API like rte_eth_dev_set_mtu be called by a normal pthread

[Antonio Di Bacco <a.dibacco.ks@gmail.com>]

Hi Dmitry,

sorry for the late reply.

I changed a lot what I was doing but the effect is the same: probing,
configuring an VF on secondary process doesn't work properly on all
eth cards.
It works with an Intel E810-C but doesn't work with an MLX5 for example.

It happens that I'm able to configure the VF on MLX5 with no errors on
secondary but when using the rte_eth_tx_burst I get a SIGSEGV:


Thread 4 "lcore-worker-6" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fbeed72d640 (LWP 11614)]
rte_eth_tx_burst (port_id=0, queue_id=0, tx_pkts=0x7fbeed72bac0,
nb_pkts=1) at /usr/local/include/rte_ethdev.h:5650
5650            qd = p->txq.data[queue_id];
(gdb) bt
#0  rte_eth_tx_burst (port_id=0, queue_id=0, tx_pkts=0x7fbeed72bac0,
nb_pkts=1) at /usr/local/include/rte_ethdev.h:5650
#1  0x0000557e11498f69 in tx (port_id=0, len=16, data=0x7fbeed72bb00
"@ABCDEFGPQRSTUVWO\273r\355\276\177") at main.c:113
#2  0x0000557e1149907d in lcore_secondary (arg=0x0) at main.c:152
#3  0x00007fbef117559a in eal_thread_loop[cold] () from
/usr/local/lib/librte_eal.so.22
#4  0x00007fbef0fccb43 in start_thread (arg=<optimized out>) at
./nptl/pthread_create.c:442
#5  0x00007fbef105ea00 in clone3 () at
../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

In previous stack trace I see that p->txq.data is NULL.

If I probe, configure the VF on primary this problem doesn't happen.

I'm attaching the code that is not polished but can give you an idea
what I'm doing (wrong).
This code is started with a command line like this, where last par is
the BDF of VF I want to use.
sudo  build/simple_eth_tx_mp -l 10-12 --file-prefix tx --allow
'0000:00:00.0' -- 0000:3b:00.2
It simply starts a secondary process that initialize the eth port and
try to send a packet. Probably I'm not configuring properly the queue
and then I get problems.


#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <inttypes.h>
#include <stdarg.h>
#include <errno.h>
#include <unistd.h>
#include <termios.h>
#include <sys/queue.h>

#include <rte_common.h>
#include <rte_memory.h>
#include <rte_launch.h>
#include <rte_eal.h>
#include <rte_per_lcore.h>
#include <rte_lcore.h>
#include <rte_debug.h>
#include <rte_branch_prediction.h>
#include <rte_ring.h>
#include <rte_log.h>
#include <rte_ethdev.h>
#include <rte_mempool.h>
#include <cmdline_rdline.h>
#include <cmdline_parse.h>
#include <cmdline_parse_string.h>
#include <cmdline_socket.h>
#include <cmdline.h>
#include <rte_memzone.h>

#define RTE_LOGTYPE_APP RTE_LOGTYPE_USER1


volatile int quit = 0;

static struct rte_mempool* pktmbuf_pool = NULL;




int rx(uint16_t port_id)
{
    struct rte_mbuf *rx_bufs[8];

    const uint16_t nb_rx = rte_eth_rx_burst(port_id, 0, rx_bufs, 8);

    if (nb_rx > 0)
    {
        int l = rte_pktmbuf_data_len(rx_bufs[0]);

        printf("Rx len %d\n", l);
        //printf("nb_rx %d\n", nb_rx);
        rte_pktmbuf_free_bulk(rx_bufs, nb_rx);
    }

}

int tx(uint16_t port_id, int len, uint8_t* data)
{
    struct rte_mbuf *tx_bufs[1];

    if(rte_pktmbuf_alloc_bulk(pktmbuf_pool, tx_bufs, 1) !=0)
    {
        printf("err %s\n", rte_strerror(rte_errno));
        return -1;
    }

    struct rte_ether_hdr *eth_hdr = (struct rte_ether_hdr
*)rte_pktmbuf_append(tx_bufs[0],
                                    sizeof(struct rte_ether_hdr));

    eth_hdr->ether_type = 0x0ff0;

    struct rte_ether_addr dst_mac_addr = { 0xce, 0xe0, 0x2c, 0x04, 0x07, 0x65 };


    rte_memcpy(&(eth_hdr->dst_addr), &dst_mac_addr, sizeof(struct
rte_ether_addr));

    struct rte_ether_addr src_mac_addr = { 0x86, 0xda, 0x92, 0x24, 0x98, 0xf8 };

    rte_memcpy(&(eth_hdr->src_addr), &src_mac_addr, sizeof(struct
rte_ether_addr));

    if (1)
    {
        for (int i = 0; i < 14; i++)
            printf("%02x", ((uint8_t*) eth_hdr)[i]);
        printf("\n");
    }

    void* p  = rte_pktmbuf_append(tx_bufs[0], len);

    if (p == NULL)
    {
        printf("append error\n");
        return -1;
    }

    rte_memcpy(p, data, len);

    const uint16_t nb_tx = rte_eth_tx_burst(port_id, 0, tx_bufs, 1);
    printf("nb_tx %d\n", nb_tx);

}


static int
lcore_primary(__rte_unused void *arg)
{
    unsigned lcore_id = rte_lcore_id();

    printf("Starting core on primary lcore %u\n", lcore_id);


    while (true)
    {
        uint8_t data[] = { 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
                         0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57 };
        //tx(0, 16, data);
        rte_delay_ms(1000);

    }
        //printf("lcore %u hugepages size %ld\n", lcore_id,
memzone->hugepage_sz);


    return 0;
}

static int
lcore_secondary(__rte_unused void *arg)
{
    unsigned lcore_id = rte_lcore_id();

    printf("Starting core on secondary %u\n", lcore_id);

    while (true)
    {
        uint8_t data[] = { 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
                         0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57 };
        tx(0, 16, data);
        rte_delay_ms(1000);

    }

    return 0;
}



int init_mbufs()
{
    printf("Allocating pktmbuf_pool\n");

    pktmbuf_pool = rte_pktmbuf_pool_create("mypool", 8191, 512, 0,
RTE_MBUF_DEFAULT_BUF_SIZE, 0);
    if (pktmbuf_pool == NULL)
    {
        printf("Cannot allocate pktmbuf_pool\n");
        return -1;
    }
    else
    {
        printf("pktmbuf_pool %p\n", pktmbuf_pool);
    }
}

int port_init(int port_id)
{
    struct rte_eth_conf     port_conf;
    int                     retval;
    struct rte_eth_dev_info dev_info;
    struct rte_eth_rxconf   rxconf;
    struct rte_eth_txconf   txconf;

    memset( &port_conf, 0, sizeof(struct rte_eth_conf) );


    retval = rte_eth_dev_info_get(port_id, &dev_info );
    if (retval != 0)
        return -1;

    uint16_t rx_ring_size = 1024;
    uint16_t tx_ring_size = 1024;

    retval = rte_eth_dev_configure(port_id, 1, 1, &port_conf );
    if (retval != 0)
        return -1;

    printf("Eth start %d\n", __LINE__);

    retval = rte_eth_dev_adjust_nb_rx_tx_desc(
                    port_id, &rx_ring_size, &tx_ring_size );
    if (retval != 0)
        return -1;


    /* Setup the RX queue (always done, it's mandatory for mlx5 at least) */
    rxconf = dev_info.default_rxconf;
    retval = rte_eth_rx_queue_setup(port_id, 0,
                                     rx_ring_size,
                                     0, &rxconf,
                                     pktmbuf_pool );
    if (retval < 0)
        return -1;

    if ( tx_ring_size ) {

        txconf          = dev_info.default_txconf;
        txconf.offloads = port_conf.txmode.offloads;
        retval = rte_eth_tx_queue_setup(port_id, 0,
                                         tx_ring_size,
                                         0, &txconf );
        if (retval < 0)
            return -1;
    }

    /* Use DPDK for always setting the largest MTU we support, with
     * no need to manual mtu configuration on system interfaces.
     * DAVIDE tested on Intel ixgbe, let's see if it works with
i40e/mlx5 TODO */
    rte_eth_dev_set_mtu( port_id, 9000 );

    retval = rte_eth_dev_start(port_id );
    if (retval < 0)
        return -1;
}


int init_vf(const char* vf)
{
        printf("Num ports %d\n", rte_eth_dev_count_avail());
        int res = rte_dev_probe(vf);
        if (res < 0) {
            printf("Cannot probe\n");
            return -1;
        }

        uint16_t port_id = 0;
        res = rte_eth_dev_get_port_by_name(vf, &port_id );
        if (res < 0) {
            printf("Port not found\n");
            return -1;
        }

        printf("Num ports %d\n", rte_eth_dev_count_avail());

        printf("Configuring port %d\n", port_id);
        init_mbufs();
        port_init(port_id);

}

int
main(int argc, char **argv)
{
    const unsigned flags = 0;

    const unsigned priv_data_sz = 0;

    int ret;
    unsigned lcore_id;

    int numa = 0;

    ret = rte_eal_init(argc, argv);
    if (ret < 0)
        rte_exit(EXIT_FAILURE, "Cannot init EAL\n");


    argc -= ret;
    argv += ret;


    const char* vf_bdf = argv[1];
    printf("VF-BDF %s\n", vf_bdf);

    /* Start of ring structure. 8< */
    if (rte_eal_process_type() == RTE_PROC_PRIMARY)
    {
        //init_vf(vf_bdf);

        int first_lcore = rte_get_next_lcore(-1, 1, 0);
        rte_eal_remote_launch(lcore_primary, NULL, first_lcore);


        char cmdline[256];
        sprintf(cmdline, "build/simple_eth_tx_mp -l 30-32
--file-prefix tx --allow '0000:00:00.0' --proc-type auto -- %s",
vf_bdf);
        system(cmdline);

        rte_delay_ms(10000);


    } else {

        printf("Secondary pid is %d, waiting if you want to attach\n",
getpid());
        sleep(10);

        init_vf(vf_bdf);

        //
        sleep(5);
        int first_lcore = rte_get_next_lcore(-1, 1, 0);
        rte_eal_remote_launch(lcore_secondary, NULL, first_lcore);


        printf("Secondary started\n");


    }


    rte_eal_mp_wait_lcore();

    /* clean up the EAL */
    rte_eal_cleanup();

    printf("Proc type %d exiting\n", rte_eal_process_type());

    return 0;
}



On Tue, Aug 23, 2022 at 2:08 PM Dmitry Kozlyuk <dmitry.kozliuk@gmail.com> wrote:
>
> 2022-08-23 13:57 (UTC+0200), Antonio Di Bacco:
> > Thank you, and what if the pthread is launched by a secondary DPDK
> > process? Should rte_eth_dev_set_mtu be working ?
> > Because I'm observing a crash.
>
> Do you have a debug stack trace?
> Does PMD that you use support setting MTU? Memif doesn't, for example.
> Check for .mtu_set in struct eth_dev_ops filled in the PMD code.