Deadlock while allocating memory from two different primaries

Deadlock while allocating memory from two different primaries

[Antonio Di Bacco]

I have a primary that spawns another primary that will only run for a
few seconds and then exit. While spawning the "son" primary (using
system call) the "father" primary continues to allocate memory using
rte_memzone_reserve().

After a few spawnings, I get a lock:

simple_mem_mp    5542 FLOCK      WRITE* 0     0          0 /dev/hugepages2M...
simple_mem_mp    5542 FLOCK      WRITE  0     0          0 /dev/hugepages2M...

During these tests I got the impression that the lock doesn't arise
from the concurrency of the calls to allocate memory but from the
concurrency of allocations and rte_eal_init of the spawned process.

This is the reference code:


#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <inttypes.h>
#include <stdarg.h>
#include <errno.h>
#include <unistd.h>
#include <termios.h>
#include <sys/queue.h>

#include <rte_common.h>
#include <rte_memory.h>
#include <rte_launch.h>
#include <rte_eal.h>
#include <rte_per_lcore.h>
#include <rte_lcore.h>
#include <rte_debug.h>
#include <rte_branch_prediction.h>
#include <rte_ring.h>
#include <rte_log.h>
#include <rte_mempool.h>
#include <cmdline_rdline.h>
#include <cmdline_parse.h>
#include <cmdline_parse_string.h>
#include <cmdline_socket.h>
#include <cmdline.h>
#include <rte_memzone.h>
#include "mp_commands.h"

#define RTE_LOGTYPE_APP RTE_LOGTYPE_USER1

volatile int quit = 0;

static int
lcore_alloc_forever(__rte_unused void *arg)
{
    unsigned lcore_id = rte_lcore_id();

    printf("Starting core on primary lcore %u\n", lcore_id);
    while (!quit)
    {

        char mem_name[64];

        sprintf(mem_name, "%s%d", "mymem", rand());
        rte_delay_ms(10);

        const struct rte_memzone* memzone =
rte_memzone_reserve_aligned(mem_name, 1024*1024, rand()%2,
RTE_MEMZONE_256KB | RTE_MEMZONE_SIZE_HINT_ONLY |
RTE_MEMZONE_IOVA_CONTIG, RTE_CACHE_LINE_SIZE);
        if (memzone == NULL)
            printf("Cannot allocate\n");
        rte_delay_ms(10);
        //printf("lcore %u hugepages size %ld\n", lcore_id,
memzone->hugepage_sz);
        rte_memzone_free(memzone);


    }

    return 0;
}

static int
lcore_alloc_time_limited(__rte_unused void *arg)
{
    unsigned lcore_id = rte_lcore_id();
    char mem_name[32];

    printf("Starting core %u \n", lcore_id);
    for (int i = 0; i < 200; i++)
    {
        sprintf(mem_name, "%s%d", "mymem", rand());
        rte_delay_ms(rand() % 20);
        const struct rte_memzone* memzone =
rte_memzone_reserve_aligned(mem_name, 1024*1024, rand()%2,
RTE_MEMZONE_256KB | RTE_MEMZONE_SIZE_HINT_ONLY |
RTE_MEMZONE_IOVA_CONTIG, RTE_CACHE_LINE_SIZE);
        if (memzone == NULL)
            printf("Cannot allocate\n");
        rte_delay_ms(rand() % 20);
        //printf("hugepages size %ld\n", memzone->hugepage_sz);
        rte_memzone_free(memzone);

    }

    printf("Exiting core %u \n", lcore_id);

    return 0;
}


int
main(int argc, char **argv)
{
    const unsigned flags = 0;

    const unsigned priv_data_sz = 0;

    int ret;
    unsigned lcore_id;

    int numa = 0;

    ret = rte_eal_init(argc, argv);
    if (ret < 0)
        rte_exit(EXIT_FAILURE, "Cannot init EAL\n");

    argc -= ret; argv += ret;


    /* Start of ring structure. 8< */
    if (rte_eal_process_type() == RTE_PROC_PRIMARY)
    {
        RTE_LOG(INFO, APP, "Finished Process Init.\n");

        int second_process = (argv[1] != NULL) && (strcmp(argv[1],
"second") == 0);

        if (!second_process)
        {
            RTE_LCORE_FOREACH_WORKER(lcore_id) {
                rte_eal_remote_launch(lcore_alloc_forever, NULL, lcore_id);
            }

            while (true)
            {
                printf("Launching process\n");
                system("build/simple_mem_mp --file-prefix bb  -l 5-15
--proc-type primary -- second");

                #define MAX_2M_PAGES    512
                struct rte_memzone* memzones[MAX_2M_PAGES];

                for (int i = 0; i < MAX_2M_PAGES; i++)
                {
                    char mem_name[32];
                    sprintf(mem_name, "%s%d", "mymem", rand());
                    memzones[i] =
rte_memzone_reserve_aligned(mem_name, 1024*1024, rand()%2,
RTE_MEMZONE_256KB | RTE_MEMZONE_SIZE_HINT_ONLY |
RTE_MEMZONE_IOVA_CONTIG, RTE_CACHE_LINE_SIZE);
                    if (memzones[i] == NULL)
                        printf("Cannot allocate\n");
                }
                rte_delay_ms(rand() % 20);

                for (int i = 0; i < MAX_2M_PAGES; i++)
                {
                    rte_memzone_free(memzones[i]);
                }

                rte_delay_ms(10000);
            }

        }
        else
        {
            RTE_LCORE_FOREACH_WORKER(lcore_id) {
                rte_eal_remote_launch(lcore_alloc_time_limited, NULL, lcore_id);
            }

        }


    } else {

        /* call lcore_recv() on every worker lcore */
        RTE_LCORE_FOREACH_WORKER(lcore_id) {
            rte_eal_remote_launch(lcore_alloc_time_limited, NULL, lcore_id);
        }

        printf("Secondary started\n");

    }

    rte_eal_mp_wait_lcore();

    /* clean up the EAL */
    rte_eal_cleanup();

    printf("Proc type %d exiting\n", rte_eal_process_type());

    return 0;
}