rte_eth_dev_socket_id return

rte_eth_dev_socket_id return

[Danil Onishchenko]

Hello,

I faced a confusing behavior of rte_eth_dev_socket_id() function. In my 
application I configured eth port and if after that I call 
rte_eth_dev_socket_id(port) it returns -1. In documentation of the 
function it is said "Returns: The NUMA socket ID to which the Ethernet 
device is connected or a default of zero if the socket could not be 
determined. -1 is returned is the port_id value is out of range.". 
However if call rte_eth_dev_is_valid_port(port) with the same port id it 
returns 1. The documentation of this function says "Returns: 0 if port 
is out of range or not attached, 1 if device is attached". It looks like 
this functions can't return -1 and 1 respectively for the same port 
number. Are there any other cases when the first one can return -1?

Below is a code sample of port initialization and functions calls

static int port_init(uint16_t port, struct rte_mempool *mbuf_pool)
{
     struct rte_eth_conf port_conf;
     int i, retval = 0;
     struct rte_ether_addr ether_addr;
     struct rte_eth_dev_info dev_info;
         struct rte_eth_txconf txconf;
         const uint16_t rx_rings = 1, tx_rings = 1;
         uint16_t nb_rxd = RX_RING_SIZE, nb_txd = TX_RING_SIZE;
         char ethaddr[32];

     printf("ETH port found %u %.4x\n", (unsigned int)port, (unsigned 
int)port);

         memset(&port_conf, 0, sizeof(struct rte_eth_conf));

     if( !rte_eth_dev_is_valid_port(port) ) {
         printf("ETH port rte_eth_dev_is_valid_port(%.4x)==0\n", 
(unsigned int)port);
         return -1;
     }

     if( (retval = rte_eth_macaddr_get(port, &ether_addr)) != 0 ) {
         printf("ETH port rte_eth_macaddr_get(%.4x) != 0\n", (unsigned 
int)port);
         return retval;
     }

     printf("ETH port %.4x ethernet addr %s\n", (unsigned int)port, 
eth_addr_to_str(&ether_addr, ethaddr, 32));

     if( (retval = rte_eth_dev_info_get(port, &dev_info)) != 0 ) {
         printf("ETH port rte_eth_dev_info_get(%.4x) != 0\n", (unsigned 
int)port);
         return retval;
     }

     printf("Driver name: %s\n", dev_info.driver_name);

         if( dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE )
                 port_conf.txmode.offloads |= 
RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;

     if( (retval = rte_eth_dev_configure(port, rx_rings, tx_rings, 
&port_conf)) != 0 ) {
         printf("ETH port rte_eth_dev_configure(%.4x) != 0\n", (unsigned 
int)port);
                 return retval;
     }

     if( (retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, 
&nb_txd)) != 0 ) {
         printf("ETH port rte_eth_dev_adjust_nb_rx_tx_desc(%.4x) != 
0\n", (unsigned int)port);
         return retval;
     }

         /* Allocate and set up 1 RX queue per Ethernet port. */
         for( i = 0; i < rx_rings; ++i ) {
                 if( (retval = rte_eth_rx_queue_setup(port, i, nb_rxd, 
rte_eth_dev_socket_id(port), NULL, mbuf_pool)) < 0 ) {
                     printf("ETH port rte_eth_rx_queue_setup(%.4x) < 
0\n", (unsigned int)port);
                         return retval;
         }
         }

         txconf = dev_info.default_txconf;
         txconf.offloads = port_conf.txmode.offloads;
         /* Allocate and set up 1 TX queue per Ethernet port. */
         for( i = 0; i < tx_rings; ++i ) {
                 if( (retval = rte_eth_tx_queue_setup(port, i, nb_txd, 
rte_eth_dev_socket_id(port), &txconf)) < 0 ) {
                     printf("ETH port rte_eth_tx_queue_setup(%.4x) < 
0\n", (unsigned int)port);
                         return retval;
         }
         }

     if( (retval = rte_eth_dev_start(port)) < 0 ) {
         printf("ETH port rte_eth_dev_start(%.4x) < 0\n", (unsigned 
int)port);
         return retval;
     }

     if( (retval = rte_eth_macaddr_get(port, &ether_addr)) != 0 ) {
         printf("ETH port rte_eth_macaddr_get(%.4x) != 0\n", (unsigned 
int)port);
         goto cleanup;
     }

     printf("ETH port %.4x ethernet addr %s\n", (unsigned int)port, 
eth_addr_to_str(&ether_addr, ethaddr, 32));

         if( (retval = rte_eth_promiscuous_enable(port)) != 0 ) {
             printf("ETH port rte_eth_promiscuous_enable(%.4x) != 0\n", 
(unsigned int)port);
             goto cleanup;
     }

     printf("PORT %u rte_eth_dev_is_valid_port=%d 
rte_eth_dev_socket_id=%d\n", (unsigned int)port, 
rte_eth_dev_is_valid_port(port), rte_eth_dev_socket_id(port));

cleanup:
     if( retval != 0 )
         if( rte_eth_dev_stop(port) != 0 )
             printf("ETH port rte_eth_dev_stop(%.4x) != 0\n", (unsigned 
int)port);

finish:
     return retval;
}

The output of the line printf("PORT %u rte_eth_dev_is_valid_port=%d 
rte_eth_dev_socket_id=%d\n", (unsigned int)port, 
rte_eth_dev_is_valid_port(port), rte_eth_dev_socket_id(port));

PORT 0 rte_eth_dev_is_valid_port=1 rte_eth_dev_socket_id=-1

Btw during the start I can see a log message about NUMA if it might help

EAL: Detected NUMA nodes: 1

Re: rte_eth_dev_socket_id return

[Dmitry Kozlyuk]

2022-08-14 15:36 (UTC+0700), Danil Onishchenko:
> Hello,
> 
> I faced a confusing behavior of rte_eth_dev_socket_id() function. In my 
> application I configured eth port and if after that I call 
> rte_eth_dev_socket_id(port) it returns -1. In documentation of the 
> function it is said "Returns: The NUMA socket ID to which the Ethernet 
> device is connected or a default of zero if the socket could not be 
> determined. -1 is returned is the port_id value is out of range.". 
> However if call rte_eth_dev_is_valid_port(port) with the same port id it 
> returns 1. The documentation of this function says "Returns: 0 if port 
> is out of range or not attached, 1 if device is attached". It looks like 
> this functions can't return -1 and 1 respectively for the same port 
> number. Are there any other cases when the first one can return -1?

Hi Danil,

This function returns whatever the bus driver discovers.
Assuming Linux and PCI, the bus driver reads
/sys/bus/pci/devices/<addr>/numa_node
Does this file contain "-1" for your device?

It's a DPDK bug anyway that documentation doesn't match behavior.

Relevant bus driver code:

	/* get numa node, default to 0 if not present */
	snprintf(filename, sizeof(filename), "%s/numa_node",
		 dirname);

	if (access(filename, F_OK) != -1) {
		if (eal_parse_sysfs_value(filename, &tmp) == 0)
			dev->device.numa_node = tmp;
		else
			dev->device.numa_node = -1;
	} else {
		dev->device.numa_node = 0;
	}

Re: rte_eth_dev_socket_id return

[Danil Onishchenko]

Hi Dmitry,

Yes, /sys/bus/pci/devices/<addr>/numa_node contains -1.

On 17.08.2022 03:08, Dmitry Kozlyuk wrote:
> 2022-08-14 15:36 (UTC+0700), Danil Onishchenko:
>> Hello,
>>
>> I faced a confusing behavior of rte_eth_dev_socket_id() function. In my
>> application I configured eth port and if after that I call
>> rte_eth_dev_socket_id(port) it returns -1. In documentation of the
>> function it is said "Returns: The NUMA socket ID to which the Ethernet
>> device is connected or a default of zero if the socket could not be
>> determined. -1 is returned is the port_id value is out of range.".
>> However if call rte_eth_dev_is_valid_port(port) with the same port id it
>> returns 1. The documentation of this function says "Returns: 0 if port
>> is out of range or not attached, 1 if device is attached". It looks like
>> this functions can't return -1 and 1 respectively for the same port
>> number. Are there any other cases when the first one can return -1?
> Hi Danil,
>
> This function returns whatever the bus driver discovers.
> Assuming Linux and PCI, the bus driver reads
> /sys/bus/pci/devices/<addr>/numa_node
> Does this file contain "-1" for your device?
>
> It's a DPDK bug anyway that documentation doesn't match behavior.
>
> Relevant bus driver code:
>
> 	/* get numa node, default to 0 if not present */
> 	snprintf(filename, sizeof(filename), "%s/numa_node",
> 		 dirname);
>
> 	if (access(filename, F_OK) != -1) {
> 		if (eal_parse_sysfs_value(filename, &tmp) == 0)
> 			dev->device.numa_node = tmp;
> 		else
> 			dev->device.numa_node = -1;
> 	} else {
> 		dev->device.numa_node = 0;
> 	}