[dpdk-dev] [PATCH 1/2] timer: use rte_mp_msg to pass TSC hz to secondary procs

[dpdk-dev] [PATCH 1/2] timer: use rte_mp_msg to pass TSC hz to secondary procs

[Jim Harris]

rte_eal_init() is much faster in secondary processes since
hugepages don't need to be zeroed.  But there's still
non-trivial delays in the timer subsystem initialization
due to the 100ms sleep used to calculate TSC hz.  So use
the rte_mp_msg framework to allow secondary processes
to get the TSC hz from the primary process.

This cuts rte_eal_init() execution time in a secondary
process from 165ms to 66ms in my test program.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
---
 lib/librte_eal/common/eal_common_timer.c |   70 +++++++++++++++++++++++++++++-
 1 file changed, 68 insertions(+), 2 deletions(-)

diff --git a/lib/librte_eal/common/eal_common_timer.c b/lib/librte_eal/common/eal_common_timer.c
index 145543de7..a2ad0f8ca 100644
--- a/lib/librte_eal/common/eal_common_timer.c
+++ b/lib/librte_eal/common/eal_common_timer.c
@@ -15,9 +15,16 @@
 #include <rte_log.h>
 #include <rte_cycles.h>
 #include <rte_pause.h>
+#include <rte_eal.h>
 
 #include "eal_private.h"
 
+#define EAL_TIMER_MP "eal_timer_mp_sync"
+
+struct timer_mp_param {
+	uint64_t tsc;
+};
+
 /* The frequency of the RDTSC timer resolution */
 static uint64_t eal_tsc_resolution_hz;
 
@@ -74,8 +81,8 @@ estimate_tsc_freq(void)
 	return RTE_ALIGN_MUL_NEAR(rte_rdtsc() - start, CYC_PER_10MHZ);
 }
 
-void
-set_tsc_freq(void)
+static void
+set_tsc_freq_primary(void)
 {
 	uint64_t freq;
 
@@ -89,6 +96,65 @@ set_tsc_freq(void)
 	eal_tsc_resolution_hz = freq;
 }
 
+static void
+set_tsc_freq_secondary(void)
+{
+	struct rte_mp_msg mp_req;
+	struct rte_mp_reply mp_reply;
+	struct timer_mp_param *r;
+	struct timespec ts = {.tv_sec = 1, .tv_nsec = 0};
+
+	memset(&mp_req, 0, sizeof(mp_req));
+	strcpy(mp_req.name, EAL_TIMER_MP);
+	if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) || mp_reply.nb_received != 1) {
+		/* We weren't able to get the tsc hz from the primary process.  So we will
+		 * just calculate it here in the secondary process instead.
+		 */
+		set_tsc_freq_primary();
+		return;
+	}
+
+	r = (struct timer_mp_param *)mp_reply.msgs[0].param;
+	eal_tsc_resolution_hz = r->tsc;
+	free(mp_reply.msgs);
+}
+
+static int
+timer_mp_primary(__attribute__((unused)) const struct rte_mp_msg *msg, const void *peer)
+{
+	struct rte_mp_msg reply;
+	struct timer_mp_param *r = (struct timer_mp_param *)reply.param;
+
+	memset(&reply, 0, sizeof(reply));
+	r->tsc = eal_tsc_resolution_hz;
+	strcpy(reply.name, EAL_TIMER_MP);
+	reply.len_param = sizeof(*r);
+
+	return rte_mp_reply(&reply, peer);
+}
+
+void
+set_tsc_freq(void)
+{
+	int rc;
+
+	/* We use a 100ms timer to calculate the TSC hz.  We can save this 100ms in
+	 * secondary processes, by getting the TSC hz from the primary process.
+	 * So register an mp_action callback in the primary process, which secondary
+	 * processes will use to get the TSC hz.
+	 */
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		set_tsc_freq_primary();
+		rc = rte_mp_action_register(EAL_TIMER_MP, timer_mp_primary);
+		if (rc) {
+			RTE_LOG(WARNING, EAL, "Could not register mp_action - secondary "
+				" processes will calculate TSC independently.\n");
+		}
+	} else {
+		set_tsc_freq_secondary();
+	}
+}
+
 void rte_delay_us_callback_register(void (*userfunc)(unsigned int))
 {
 	rte_delay_us = userfunc;

[dpdk-dev] [PATCH 2/2] timer: don't check tsc flags in secondary processes

[Jim Harris]

check_tsc_flags() parses /proc/cpuinfo and prints
warning messages if any cores don't have constant_tsc
and nonstop_tsc.  It has no functional meaning.
This consumes a noticeable amount of time in
secondary processes - on my test system, it consumes
21ms out of the 66ms total execution time for
rte_eal_init().

So let's just skip checking these flags in secondary
processes.  Since the primary process is already
parsing the entirety of /proc/cpuinfo, the warning
printed in the primary process should be sufficient.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
---
 lib/librte_eal/linux/eal/eal_timer.c |    9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/lib/librte_eal/linux/eal/eal_timer.c b/lib/librte_eal/linux/eal/eal_timer.c
index 76ec17034..ce447d43b 100644
--- a/lib/librte_eal/linux/eal/eal_timer.c
+++ b/lib/librte_eal/linux/eal/eal_timer.c
@@ -198,6 +198,15 @@ check_tsc_flags(void)
 	char line[512];
 	FILE *stream;
 
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		/* This function just prints warnings if TSC is not constant
+		 * and has no functional meaning.  It also checks *all* cores
+		 * on the system, not just the ones configured for this process.
+		 * So don't bother rechecking again in secondary processes.
+		 */
+		return;
+	}
+
 	stream = fopen("/proc/cpuinfo", "r");
 	if (!stream) {
 		RTE_LOG(WARNING, EAL, "WARNING: Unable to open /proc/cpuinfo\n");

Re: [dpdk-dev] [PATCH 1/2] timer: use rte_mp_msg to pass TSC hz to secondary procs

[Burakov, Anatoly]

On 15-Aug-19 12:27 PM, Jim Harris wrote:
> rte_eal_init() is much faster in secondary processes since
> hugepages don't need to be zeroed.  But there's still
> non-trivial delays in the timer subsystem initialization
> due to the 100ms sleep used to calculate TSC hz.  So use
> the rte_mp_msg framework to allow secondary processes
> to get the TSC hz from the primary process.
> 
> This cuts rte_eal_init() execution time in a secondary
> process from 165ms to 66ms in my test program.
> 
> Signed-off-by: Jim Harris <james.r.harris@intel.com>
> ---

<snip>

> @@ -89,6 +96,65 @@ set_tsc_freq(void)
>   	eal_tsc_resolution_hz = freq;
>   }
>   
> +static void
> +set_tsc_freq_secondary(void)
> +{
> +	struct rte_mp_msg mp_req;
> +	struct rte_mp_reply mp_reply;
> +	struct timer_mp_param *r;
> +	struct timespec ts = {.tv_sec = 1, .tv_nsec = 0};
> +
> +	memset(&mp_req, 0, sizeof(mp_req));
> +	strcpy(mp_req.name, EAL_TIMER_MP);
> +	if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) || mp_reply.nb_received != 1) {

If rte_mp_request_sync returns 0 but mp_reply.nb_receieved isn't set to 
1, you'll be leaking mp_reply.msgs.

-- 
Thanks,
Anatoly

Re: [dpdk-dev] [PATCH 1/2] timer: use rte_mp_msg to pass TSC hz to secondary procs

[Harris, James R]

On 8/16/19, 12:56 AM, "Burakov, Anatoly" <anatoly.burakov@intel.com> wrote:

    
    <snip>
    
    > @@ -89,6 +96,65 @@ set_tsc_freq(void)
    >   	eal_tsc_resolution_hz = freq;
    >   }
    >   
    > +static void
    > +set_tsc_freq_secondary(void)
    > +{
    > +	struct rte_mp_msg mp_req;
    > +	struct rte_mp_reply mp_reply;
    > +	struct timer_mp_param *r;
    > +	struct timespec ts = {.tv_sec = 1, .tv_nsec = 0};
    > +
    > +	memset(&mp_req, 0, sizeof(mp_req));
    > +	strcpy(mp_req.name, EAL_TIMER_MP);
    > +	if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) || mp_reply.nb_received != 1) {
    
    If rte_mp_request_sync returns 0 but mp_reply.nb_receieved isn't set to 
    1, you'll be leaking mp_reply.msgs.


Ha - of course you're right.  I didn't notice this when using the VFIO code as a reference for
how to code this.  I'll respin this patch and then push patches to fix the VFIO code separately.

Thanks,

-Jim

Re: [dpdk-dev] [PATCH 1/2] timer: use rte_mp_msg to pass TSC hz to secondary procs

[Bruce Richardson]

On Thu, Aug 15, 2019 at 04:27:35AM -0700, Jim Harris wrote:
> rte_eal_init() is much faster in secondary processes since
> hugepages don't need to be zeroed.  But there's still
> non-trivial delays in the timer subsystem initialization
> due to the 100ms sleep used to calculate TSC hz.  So use
> the rte_mp_msg framework to allow secondary processes
> to get the TSC hz from the primary process.
> 
> This cuts rte_eal_init() execution time in a secondary
> process from 165ms to 66ms in my test program.
> 
> Signed-off-by: Jim Harris <james.r.harris@intel.com>
> ---

Rather than messaging, can we not just move the CPU frequency to being
stored in a shared memory location? It's not something where different
processes are going to need to be provided with different values.