Re: [dpdk-dev] [PATCH v3 1/2] timer: use rte_mp_msg to pass TSC hz to secondary procs

["Harris, James R" <james.r.harris@intel.com>]

On 8/16/19, 12:19 PM, "dev on behalf of Jim Harris" <dev-bounces@dpdk.org on behalf of james.r.harris@intel.com> wrote:

    rte_eal_init() is much faster in secondary processes since
    hugepages don't need to be zeroed.  But there's still
    non-trivial delays in the timer subsystem initialization
    due to the 100ms sleep used to calculate TSC hz.  So use
    the rte_mp_msg framework to allow secondary processes
    to get the TSC hz from the primary process.
    
    This cuts rte_eal_init() execution time in a secondary
    process from 165ms to 66ms in my test program.

I'm withdrawing this patch.  Some Linux distros (Ubuntu) build msr as a module and if it's not loaded,
will go down the sleep path.  The (much) simpler solution is to just modprobe msr.



    Signed-off-by: Jim Harris <james.r.harris@intel.com>
    ---
     lib/librte_eal/common/eal_common_timer.c |   70 +++++++++++++++++++++++++++++-
     1 file changed, 68 insertions(+), 2 deletions(-)
    
    diff --git a/lib/librte_eal/common/eal_common_timer.c b/lib/librte_eal/common/eal_common_timer.c
    index 145543de7..3eb0309f5 100644
    --- a/lib/librte_eal/common/eal_common_timer.c
    +++ b/lib/librte_eal/common/eal_common_timer.c
    @@ -15,9 +15,16 @@
     #include <rte_log.h>
     #include <rte_cycles.h>
     #include <rte_pause.h>
    +#include <rte_eal.h>
     
     #include "eal_private.h"
     
    +#define EAL_TIMER_MP "eal_timer_mp_sync"
    +
    +struct timer_mp_param {
    +	uint64_t tsc;
    +};
    +
     /* The frequency of the RDTSC timer resolution */
     static uint64_t eal_tsc_resolution_hz;
     
    @@ -74,8 +81,8 @@ estimate_tsc_freq(void)
     	return RTE_ALIGN_MUL_NEAR(rte_rdtsc() - start, CYC_PER_10MHZ);
     }
     
    -void
    -set_tsc_freq(void)
    +static void
    +set_tsc_freq_primary(void)
     {
     	uint64_t freq;
     
    @@ -89,6 +96,65 @@ set_tsc_freq(void)
     	eal_tsc_resolution_hz = freq;
     }
     
    +static void
    +set_tsc_freq_secondary(void)
    +{
    +	struct rte_mp_msg mp_req = {0};
    +	struct rte_mp_reply mp_reply = {0};
    +	struct timer_mp_param *r;
    +	struct timespec ts = {.tv_sec = 1, .tv_nsec = 0};
    +
    +	strcpy(mp_req.name, EAL_TIMER_MP);
    +	if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) || mp_reply.nb_received != 1) {
    +		/* We weren't able to get the tsc hz from the primary process.  So we will
    +		 * just calculate it here in the secondary process instead.
    +		 */
    +		set_tsc_freq_primary();
    +		free(mp_reply.msgs);
    +		return;
    +	}
    +
    +	r = (struct timer_mp_param *)mp_reply.msgs[0].param;
    +	eal_tsc_resolution_hz = r->tsc;
    +	free(mp_reply.msgs);
    +}
    +
    +static int
    +timer_mp_primary(__attribute__((unused)) const struct rte_mp_msg *msg, const void *peer)
    +{
    +	struct rte_mp_msg reply;
    +	struct timer_mp_param *r = (struct timer_mp_param *)reply.param;
    +
    +	memset(&reply, 0, sizeof(reply));
    +	r->tsc = eal_tsc_resolution_hz;
    +	strcpy(reply.name, EAL_TIMER_MP);
    +	reply.len_param = sizeof(*r);
    +
    +	return rte_mp_reply(&reply, peer);
    +}
    +
    +void
    +set_tsc_freq(void)
    +{
    +	int rc;
    +
    +	/* We use a 100ms timer to calculate the TSC hz.  We can save this 100ms in
    +	 * secondary processes, by getting the TSC hz from the primary process.
    +	 * So register an mp_action callback in the primary process, which secondary
    +	 * processes will use to get the TSC hz.
    +	 */
    +	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
    +		set_tsc_freq_primary();
    +		rc = rte_mp_action_register(EAL_TIMER_MP, timer_mp_primary);
    +		if (rc) {
    +			RTE_LOG(WARNING, EAL, "Could not register mp_action - secondary "
    +				" processes will calculate TSC independently.\n");
    +		}
    +	} else {
    +		set_tsc_freq_secondary();
    +	}
    +}
    +
     void rte_delay_us_callback_register(void (*userfunc)(unsigned int))
     {
     	rte_delay_us = userfunc;