From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from homiemail-a31.g.dreamhost.com (sub4.mail.dreamhost.com [69.163.253.135]) by dpdk.org (Postfix) with ESMTP id 116C7558D for ; Wed, 9 Nov 2016 18:12:56 +0100 (CET) Received: from homiemail-a31.g.dreamhost.com (localhost [127.0.0.1]) by homiemail-a31.g.dreamhost.com (Postfix) with ESMTP id ABACB1406B0B for ; Wed, 9 Nov 2016 09:12:55 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha1; c=relaxed; d=aclectic.com; h= content-type:mime-version:subject:from:in-reply-to:date :content-transfer-encoding:message-id:references:to; s= aclectic.com; bh=29z88D7FavG2ZdRubSGia0qAfLE=; b=HgJPJNEHMRap1bZ fPTi5rRJPbzDq6GDBb2eziPQvkOI+3w1rYJk4U/lKmefIT/nNGfgrSb61fQ4HNYA 6A58c4g+hQ9vdMN9t3wxDmGJEeOjRR591ncB/clKDPOEjL1WOZFosQHEXiEGVdm/ sl5e9wsfp9FwxoTlqnz50AsgRP3g= Received: from [192.168.42.133] (184-23-16-60.dsl.static.fusionbroadband.com [184.23.16.60]) (using TLSv1 with cipher ECDHE-RSA-AES256-SHA (256/256 bits)) (No client certificate requested) (Authenticated sender: jason.lefley@aclectic.com) by homiemail-a31.g.dreamhost.com (Postfix) with ESMTPSA id 7655D1406B0A for ; Wed, 9 Nov 2016 09:12:55 -0800 (PST) Content-Type: text/plain; charset=utf-8 Mime-Version: 1.0 (Mac OS X Mail 9.3 \(3124\)) From: Jason Lefley In-Reply-To: Date: Wed, 9 Nov 2016 09:12:55 -0800 Content-Transfer-Encoding: quoted-printable Message-Id: References: To: users@dpdk.org X-Mailer: Apple Mail (2.3124) Subject: Re: [dpdk-users] Using DPDK EAL with existing multi-threaded application X-BeenThere: users@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: usage discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 09 Nov 2016 17:12:57 -0000 I commented out code in rte_eal_init() relating to threading: int rte_eal_init(int argc, char **argv) { int i, fctret, ret; // pthread_t thread_id; // static rte_atomic32_t run_once =3D RTE_ATOMIC32_INIT(0); const char *logid; // char cpuset[RTE_CPU_AFFINITY_STR_LEN]; // char thread_name[RTE_MAX_THREAD_NAME_LEN]; // if (!rte_atomic32_test_and_set(&run_once)) // return -1; logid =3D strrchr(argv[0], '/'); logid =3D strdup(logid ? logid + 1: argv[0]); // thread_id =3D pthread_self(); if (rte_eal_log_early_init() < 0) rte_panic("Cannot init early logs\n"); eal_log_level_parse(argc, argv); /* set log level as early as possible */ rte_set_log_level(internal_config.log_level); if (rte_eal_cpu_init() < 0) rte_panic("Cannot detect lcores\n"); fctret =3D eal_parse_args(argc, argv); if (fctret < 0) exit(1); if (internal_config.no_hugetlbfs =3D=3D 0 && internal_config.process_type !=3D RTE_PROC_SECONDARY && internal_config.xen_dom0_support =3D=3D 0 && eal_hugepage_info_init() < 0) rte_panic("Cannot get hugepage information\n"); if (internal_config.memory =3D=3D 0 && internal_config.force_sockets = =3D=3D 0) { if (internal_config.no_hugetlbfs) internal_config.memory =3D MEMSIZE_IF_NO_HUGE_PAGE; } if (internal_config.vmware_tsc_map =3D=3D 1) { #ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT rte_cycles_vmware_tsc_map =3D 1; RTE_LOG (DEBUG, EAL, "Using VMWARE TSC MAP, " "you must have monitor_control.pseudo_perfctr =3D TRUE\n"); #else RTE_LOG (WARNING, EAL, "Ignoring --vmware-tsc-map because " "RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT is not set\n"); #endif } rte_srand(rte_rdtsc()); rte_config_init(); // if (rte_eal_pci_init() < 0) // rte_panic("Cannot init PCI\n"); // //#ifdef VFIO_PRESENT // if (rte_eal_vfio_setup() < 0) // rte_panic("Cannot init VFIO\n"); //#endif // //#ifdef RTE_LIBRTE_IVSHMEM // if (rte_eal_ivshmem_init() < 0) // rte_panic("Cannot init IVSHMEM\n"); //#endif if (rte_eal_memory_init() < 0) rte_panic("Cannot init memory\n"); /* the directories are locked during eal_hugepage_info_init */ eal_hugedirs_unlock(); if (rte_eal_memzone_init() < 0) rte_panic("Cannot init memzone\n"); if (rte_eal_tailqs_init() < 0) rte_panic("Cannot init tail queues for objects\n"); //#ifdef RTE_LIBRTE_IVSHMEM // if (rte_eal_ivshmem_obj_init() < 0) // rte_panic("Cannot init IVSHMEM objects\n"); //#endif if (rte_eal_log_init(logid, internal_config.syslog_facility) < 0) rte_panic("Cannot init logs\n"); // // if (rte_eal_alarm_init() < 0) // rte_panic("Cannot init interrupt-handling thread\n"); if (rte_eal_timer_init() < 0) rte_panic("Cannot init HPET or TSC timers\n"); // eal_check_mem_on_local_socket(); // if (eal_plugins_init() < 0) // rte_panic("Cannot init plugins\n"); // eal_thread_init_master(rte_config.master_lcore); // ret =3D eal_thread_dump_affinity(cpuset, RTE_CPU_AFFINITY_STR_LEN); // RTE_LOG(DEBUG, EAL, "Master lcore %u is ready = (tid=3D%x;cpuset=3D[%s%s])\n", // rte_config.master_lcore, (int)thread_id, cpuset, // ret =3D=3D 0 ? "" : "..."); // if (rte_eal_dev_init() < 0) // rte_panic("Cannot init pmd devices\n"); // if (rte_eal_intr_init() < 0) // rte_panic("Cannot init interrupt-handling thread\n"); // RTE_LCORE_FOREACH_SLAVE(i) { // // /* // * create communication pipes between master thread // * and children // */ // if (pipe(lcore_config[i].pipe_master2slave) < 0) // rte_panic("Cannot create pipe\n"); // if (pipe(lcore_config[i].pipe_slave2master) < 0) // rte_panic("Cannot create pipe\n"); // // lcore_config[i].state =3D WAIT; // // /* create a thread for each lcore */ // ret =3D pthread_create(&lcore_config[i].thread_id, NULL, // eal_thread_loop, NULL); // if (ret !=3D 0) // rte_panic("Cannot create thread\n"); // // /* Set thread_name for aid in debugging. */ // snprintf(thread_name, RTE_MAX_THREAD_NAME_LEN, // "lcore-slave-%d", i); // ret =3D rte_thread_setname(lcore_config[i].thread_id, // thread_name); // if (ret !=3D 0) // RTE_LOG(DEBUG, EAL, // "Cannot set name for lcore thread\n"); // } // /* // * Launch a dummy function on all slave lcores, so that master lcore // * knows they are all ready when this function returns. // */ // rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER); // rte_eal_mp_wait_lcore(); // /* Probe & Initialize PCI devices */ // if (rte_eal_pci_probe()) // rte_panic("Cannot probe PCI\n"); rte_eal_mcfg_complete(); return fctret; } After recompiling, I do not have the same issue when using TBB in the = same application. This works for now but I am interested in = understanding more about why this fixes the issue. It also seems like = rte_eal_init() initializes a bunch of sub-systems. Perhaps more = granularity at this level could be useful in addition to allow a more = flexible initialization API that allows more programatic control instead = of only allowing initialization from argv. > On Nov 7, 2016, at 10:39 PM, Jason Lefley = wrote: >=20 > I am working on leveraging SPDK (which internally uses EAL from DPDK) = from an existing application that uses Intel=E2=80=99s TBB. I do not = need to use EAL for multithreading but SPDK does require at least huge = page support and PCI access from EAL. >=20 > I noticed that if I try to use TBB=E2=80=99s parallel_for() after I = have called rte_eal_init(), then the TBB operation doesn=E2=80=99t = actually run in parallel. If I remove the call to rte_eal_init(), the = TBB operations work as expected however I then cannot use SPDK. Does = anyone have any input regarding how I can keep my existing = multi-threaded application working as intended as I integrate SPDK/DPDK? >=20 > Thanks