Re: [dpdk-dev] [PATCH] add one option memory-only for those secondary PRBs

["Chi, Xiaobo (NSN - CN/Hangzhou)" <xiaobo.chi@nsn.com>]

Hi, 
The similar functionality of this patch has been sent out for review on Nov.12, email topic is "[dpdk-dev] one lightwight rte_eal_init() for SECONDARY processes which only use sharedmemory".  This time, I just change the implementation method according to comments from Bruce Richardson. 

Background:
What we are doing now is port make telecom network element to be cloud based.  For one of our product,  DPDK is applied not only for fastpath/dataplane processing, but also for Distributed Message eXchange (DMX) between different processes/applications which may located in different VM even different host.  for such a DMX system, in one VM, we have one DPDK based dmxdemo (which acts as the PRIMARY) which is in charge of distribute message between different applications, and dozens of applications (act as SECONDARY) to use DPDK based rte_tx_ring/rte_rx_ring/mempool/memzone to send receive messages to dmxdemo.

Problem:
Here, these DPDK based SECONDARY processes need only the DPDK's hugepage based sharememory mechanism and it's upper libs (such as ring, mempool, etc.), they need not cpu core pinning, iopl privilege changing , pci device, timer, alarm, interrupt, shared_driver_list,  core_info, threads for each core, etc. Then, for such kind of SECONDARY processes, the current rte_eal_init() is too heavy.  
I have seen some others also met similar troubles. 

Solution:
One new EAL initializing argument, --memory-only, is added. It is only for those SECONDARY processes which only want to share memory with primary process. When this argument is defined, users need not define those madentory arguments, such as "-c xx -n xx", due to we don't want to pin such kind of processes to any CPUs.

I have tested on my test env, it works. Please kindly help to review and give comments. Thanks.

-----Original Message-----
From: chixiaobo [mailto:xiaobo.chi@nsn.com] 
Sent: Tuesday, December 02, 2014 5:41 PM
To: dev@dpdk.org
Cc: Chi, Xiaobo (NSN - CN/Hangzhou)
Subject: [PATCH] add one option memory-only for those secondary PRBs

---
 lib/librte_eal/common/eal_common_options.c |  18 +++-
 lib/librte_eal/common/eal_internal_cfg.h   |   1 +
 lib/librte_eal/common/eal_options.h        |   4 +-
 lib/librte_eal/linuxapp/eal/eal.c          | 137 +++++++++++++++--------------
 4 files changed, 89 insertions(+), 71 deletions(-)
 mode change 100644 => 100755 lib/librte_eal/common/eal_common_options.c
 mode change 100644 => 100755 lib/librte_eal/common/eal_internal_cfg.h
 mode change 100644 => 100755 lib/librte_eal/common/eal_options.h
 mode change 100644 => 100755 lib/librte_eal/linuxapp/eal/eal.c

diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
old mode 100644
new mode 100755
index e2810ab..5ab4b87
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -85,6 +85,7 @@ eal_long_options[] = {
 	{OPT_XEN_DOM0, 0, 0, OPT_XEN_DOM0_NUM},
 	{OPT_CREATE_UIO_DEV, 1, NULL, OPT_CREATE_UIO_DEV_NUM},
 	{OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM},
+    {OPT_MEMORY_ONLY, 0, NULL, OPT_MEMORY_ONLY_NUM},
 	{0, 0, 0, 0}
 };
 
@@ -126,6 +127,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 	internal_cfg->no_hpet = 1;
 #endif
 	internal_cfg->vmware_tsc_map = 0;
+    internal_cfg->memory_only= 0;
 }
 
 /*
@@ -454,6 +456,10 @@ eal_parse_common_option(int opt, const char *optarg,
 		conf->process_type = eal_parse_proc_type(optarg);
 		break;
 
+	case OPT_MEMORY_ONLY_NUM:
+		conf->memory_only= 1;
+		break;
+
 	case OPT_MASTER_LCORE_NUM:
 		if (eal_parse_master_lcore(optarg) < 0) {
 			RTE_LOG(ERR, EAL, "invalid parameter for --"
@@ -525,9 +531,9 @@ eal_check_common_options(struct internal_config *internal_cfg)
 {
 	struct rte_config *cfg = rte_eal_get_configuration();
 
-	if (!lcores_parsed) {
-		RTE_LOG(ERR, EAL, "CPU cores must be enabled with options "
-			"-c or -l\n");
+	if (!lcores_parsed && !(internal_cfg->process_type == RTE_PROC_SECONDARY&& internal_cfg->memory_only) ) {
+		RTE_LOG(ERR, EAL, "For those processes without memory-only option, CPU cores "
+            "must be enabled with options -c or -l\n");
 		return -1;
 	}
 	if (cfg->lcore_role[cfg->master_lcore] != ROLE_RTE) {
@@ -545,6 +551,11 @@ eal_check_common_options(struct internal_config *internal_cfg)
 			"specified\n");
 		return -1;
 	}
+	if ( internal_cfg->process_type != RTE_PROC_SECONDARY &&
+			internal_cfg->memory_only) {
+		RTE_LOG(ERR, EAL, "only secondary processes can specify memory-only option.\n");
+		return -1;
+	}
 	if (index(internal_cfg->hugefile_prefix, '%') != NULL) {
 		RTE_LOG(ERR, EAL, "Invalid char, '%%', in --"OPT_FILE_PREFIX" "
 			"option\n");
@@ -590,6 +601,7 @@ eal_common_usage(void)
 	       "  --"OPT_SYSLOG"     : set syslog facility\n"
 	       "  --"OPT_LOG_LEVEL"  : set default log level\n"
 	       "  --"OPT_PROC_TYPE"  : type of this process\n"
+           "  --"OPT_MEMORY_ONLY": only use shared memory, valid only for secondary process."
 	       "  --"OPT_PCI_BLACKLIST", -b: add a PCI device in black list.\n"
 	       "               Prevent EAL from using this PCI device. The argument\n"
 	       "               format is <domain:bus:devid.func>.\n"
diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h
old mode 100644
new mode 100755
index aac6abf..68b982c
--- a/lib/librte_eal/common/eal_internal_cfg.h
+++ b/lib/librte_eal/common/eal_internal_cfg.h
@@ -85,6 +85,7 @@ struct internal_config {
 
 	unsigned num_hugepage_sizes;      /**< how many sizes on this system */
 	struct hugepage_info hugepage_info[MAX_HUGEPAGE_SIZES];
+    volatile unsigned memory_only;    /**<wheter the seconday process only need shared momory only or not */
 };
 extern struct internal_config internal_config; /**< Global EAL configuration. */
 
diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h
old mode 100644
new mode 100755
index e476f8d..6ddff1b
--- a/lib/librte_eal/common/eal_options.h
+++ b/lib/librte_eal/common/eal_options.h
@@ -76,7 +76,9 @@ enum {
 #define OPT_CREATE_UIO_DEV "create-uio-dev"
 	OPT_CREATE_UIO_DEV_NUM,
 #define OPT_VFIO_INTR    "vfio-intr"
-	OPT_VFIO_INTR_NUM,
+        OPT_VFIO_INTR_NUM,
+#define OPT_MEMORY_ONLY  "memory-only"
+        OPT_MEMORY_ONLY_NUM,
 	OPT_LONG_MAX_NUM
 };
 
diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
old mode 100644
new mode 100755
index 89f3b5e..a03e511
--- a/lib/librte_eal/linuxapp/eal/eal.c
+++ b/lib/librte_eal/linuxapp/eal/eal.c
@@ -752,14 +752,16 @@ rte_eal_init(int argc, char **argv)
 
 	rte_config_init();
 
-	if (rte_eal_pci_init() < 0)
-		rte_panic("Cannot init PCI\n");
+    /*with memory-only option, we need not cpu affinity, pci device, alarm, external devices, interrupt, etc. */
+    if( !internal_config.memory_only ){
+    	if (rte_eal_pci_init() < 0)
+    		rte_panic("Cannot init PCI\n");
 
 #ifdef RTE_LIBRTE_IVSHMEM
 	if (rte_eal_ivshmem_init() < 0)
 		rte_panic("Cannot init IVSHMEM\n");
 #endif
-
+    }
 	if (rte_eal_memory_init() < 0)
 		rte_panic("Cannot init memory\n");
 
@@ -772,73 +774,73 @@ rte_eal_init(int argc, char **argv)
 	if (rte_eal_tailqs_init() < 0)
 		rte_panic("Cannot init tail queues for objects\n");
 
-#ifdef RTE_LIBRTE_IVSHMEM
-	if (rte_eal_ivshmem_obj_init() < 0)
-		rte_panic("Cannot init IVSHMEM objects\n");
-#endif
-
 	if (rte_eal_log_init(logid, internal_config.syslog_facility) < 0)
 		rte_panic("Cannot init logs\n");
 
-	if (rte_eal_alarm_init() < 0)
-		rte_panic("Cannot init interrupt-handling thread\n");
-
-	if (rte_eal_intr_init() < 0)
-		rte_panic("Cannot init interrupt-handling thread\n");
-
-	if (rte_eal_timer_init() < 0)
-		rte_panic("Cannot init HPET or TSC timers\n");
-
-	eal_check_mem_on_local_socket();
-
-	rte_eal_mcfg_complete();
-
-	TAILQ_FOREACH(solib, &solib_list, next) {
-		RTE_LOG(INFO, EAL, "open shared lib %s\n", solib->name);
-		solib->lib_handle = dlopen(solib->name, RTLD_NOW);
-		if (solib->lib_handle == NULL)
-			RTE_LOG(WARNING, EAL, "%s\n", dlerror());
-	}
-
-	eal_thread_init_master(rte_config.master_lcore);
-
-	RTE_LOG(DEBUG, EAL, "Master core %u is ready (tid=%x)\n",
-		rte_config.master_lcore, (int)thread_id);
-
-	if (rte_eal_dev_init() < 0)
-		rte_panic("Cannot init pmd devices\n");
-
-	RTE_LCORE_FOREACH_SLAVE(i) {
-
-		/*
-		 * create communication pipes between master thread
-		 * and children
-		 */
-		if (pipe(lcore_config[i].pipe_master2slave) < 0)
-			rte_panic("Cannot create pipe\n");
-		if (pipe(lcore_config[i].pipe_slave2master) < 0)
-			rte_panic("Cannot create pipe\n");
-
-		lcore_config[i].state = WAIT;
-
-		/* create a thread for each lcore */
-		ret = pthread_create(&lcore_config[i].thread_id, NULL,
-				     eal_thread_loop, NULL);
-		if (ret != 0)
-			rte_panic("Cannot create thread\n");
-	}
-
-	/*
-	 * Launch a dummy function on all slave lcores, so that master lcore
-	 * knows they are all ready when this function returns.
-	 */
-	rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER);
-	rte_eal_mp_wait_lcore();
-
-	/* Probe & Initialize PCI devices */
-	if (rte_eal_pci_probe())
-		rte_panic("Cannot probe PCI\n");
-
+    if( !internal_config.memory_only ){
+#ifdef RTE_LIBRTE_IVSHMEM
+        if (rte_eal_ivshmem_obj_init() < 0)
+            rte_panic("Cannot init IVSHMEM objects\n");
+#endif        
+        if (rte_eal_alarm_init() < 0)
+    		rte_panic("Cannot init interrupt-handling thread\n");
+
+    	if (rte_eal_intr_init() < 0)
+    		rte_panic("Cannot init interrupt-handling thread\n");
+
+    	if (rte_eal_timer_init() < 0)
+    		rte_panic("Cannot init HPET or TSC timers\n");
+
+    	eal_check_mem_on_local_socket();
+
+    	rte_eal_mcfg_complete();
+
+    	TAILQ_FOREACH(solib, &solib_list, next) {
+    		RTE_LOG(INFO, EAL, "open shared lib %s\n", solib->name);
+    		solib->lib_handle = dlopen(solib->name, RTLD_NOW);
+    		if (solib->lib_handle == NULL)
+    			RTE_LOG(WARNING, EAL, "%s\n", dlerror());
+    	}
+
+    	eal_thread_init_master(rte_config.master_lcore);
+
+    	RTE_LOG(DEBUG, EAL, "Master core %u is ready (tid=%x)\n",
+    		rte_config.master_lcore, (int)thread_id);
+
+    	if (rte_eal_dev_init() < 0)
+    		rte_panic("Cannot init pmd devices\n");
+
+    	RTE_LCORE_FOREACH_SLAVE(i) {
+
+    		/*
+        		 * create communication pipes between master thread
+        		 * and children
+        		 */
+    		if (pipe(lcore_config[i].pipe_master2slave) < 0)
+    			rte_panic("Cannot create pipe\n");
+    		if (pipe(lcore_config[i].pipe_slave2master) < 0)
+    			rte_panic("Cannot create pipe\n");
+
+    		lcore_config[i].state = WAIT;
+
+    		/* create a thread for each lcore */
+    		ret = pthread_create(&lcore_config[i].thread_id, NULL,
+    				     eal_thread_loop, NULL);
+    		if (ret != 0)
+    			rte_panic("Cannot create thread\n");
+    	}
+
+    	/*
+        	 * Launch a dummy function on all slave lcores, so that master lcore
+        	 * knows they are all ready when this function returns.
+        	 */
+        	rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER);
+        	rte_eal_mp_wait_lcore();
+
+    	/* Probe & Initialize PCI devices */
+    	if (rte_eal_pci_probe())
+    		rte_panic("Cannot probe PCI\n");
+    }
 	return fctret;
 }
 
@@ -859,3 +861,4 @@ int rte_eal_has_hugepages(void)
 {
 	return ! internal_config.no_hugetlbfs;
 }
+
-- 
1.9.4.msysgit.2