If the user tries to reset/stop some other timer in it's callback function, which is also about to expire, using rte_timer_reset_sync/rte_timer_stop_sync the application goes into an infinite loop. This happens because rte_timer_reset_sync/rte_timer_stop_sync loop until the timer resets/stops and there is check inside timer_set_config_state which prevents a running timer from being reset/stopped by not it's own timer_cb. Therefore timer_set_config_state returns -1 due to which rte_timer_reset returns -1 and rte_timer_reset_sync goes into an infinite loop. The soloution to this problem is to return -1 from rte_timer_reset_sync/rte_timer_stop_sync in case the user tries to reset/stop some other timer in it's callback function. Bugzilla ID: 491 Fixes: 20d159f20543 ("timer: fix corruption with reset") Cc: h.mikita89@gmail.com Signed-off-by: Sarosh Arif <sarosh.arif@emumba.com> --- lib/librte_timer/rte_timer.c | 25 +++++++++++++++++++++++-- lib/librte_timer/rte_timer.h | 4 ++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/lib/librte_timer/rte_timer.c b/lib/librte_timer/rte_timer.c index 6d19ce469..ed48bcbfd 100644 --- a/lib/librte_timer/rte_timer.c +++ b/lib/librte_timer/rte_timer.c @@ -576,14 +576,24 @@ rte_timer_alt_reset(uint32_t timer_data_id, struct rte_timer *tim, } /* loop until rte_timer_reset() succeed */ -void +int rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks, enum rte_timer_type type, unsigned tim_lcore, rte_timer_cb_t fct, void *arg) { + struct rte_timer_data *timer_data; + TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data, -EINVAL); + + if (tim->status.state == RTE_TIMER_RUNNING && \ + (tim->status.owner != (uint16_t)tim_lcore || \ + tim != timer_data->priv_timer[tim_lcore].running_tim)) + return -1; + while (rte_timer_reset(tim, ticks, type, tim_lcore, fct, arg) != 0) rte_pause(); + + return 0; } static int @@ -642,11 +652,22 @@ rte_timer_alt_stop(uint32_t timer_data_id, struct rte_timer *tim) } /* loop until rte_timer_stop() succeed */ -void +int rte_timer_stop_sync(struct rte_timer *tim) { + struct rte_timer_data *timer_data; + TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data, -EINVAL); + unsigned int lcore_id = rte_lcore_id(); + + if (tim->status.state == RTE_TIMER_RUNNING && \ + (tim->status.owner != (uint16_t)lcore_id || \ + tim != timer_data->priv_timer[lcore_id].running_tim)) + return -1; + while (rte_timer_stop(tim) != 0) rte_pause(); + + return 0; } /* Test the PENDING status of the timer handle tim */ diff --git a/lib/librte_timer/rte_timer.h b/lib/librte_timer/rte_timer.h index c6b3d450d..392ca423d 100644 --- a/lib/librte_timer/rte_timer.h +++ b/lib/librte_timer/rte_timer.h @@ -275,7 +275,7 @@ int rte_timer_reset(struct rte_timer *tim, uint64_t ticks, * @param arg * The user argument of the callback function. */ -void +int rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks, enum rte_timer_type type, unsigned tim_lcore, rte_timer_cb_t fct, void *arg); @@ -314,7 +314,7 @@ int rte_timer_stop(struct rte_timer *tim); * @param tim * The timer handle. */ -void rte_timer_stop_sync(struct rte_timer *tim); +int rte_timer_stop_sync(struct rte_timer *tim); /** * Test if a timer is pending. -- 2.17.1
On Tue, 7 Jul 2020 14:03:20 +0500
Sarosh Arif <sarosh.arif@emumba.com> wrote:
> + struct rte_timer_data *timer_data;
> + TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data, -EINVAL);
> +
> + if (tim->status.state == RTE_TIMER_RUNNING && \
> + (tim->status.owner != (uint16_t)tim_lcore || \
> + tim != timer_data->priv_timer[tim_lcore].running_tim))
> + return -1;
> +
This is code not a macro.
Look at checkpatch output, no \ in code.
If the user tries to reset/stop some other timer in it's callback function, which is also about to expire, using rte_timer_reset_sync/rte_timer_stop_sync the application goes into an infinite loop. This happens because rte_timer_reset_sync/rte_timer_stop_sync loop until the timer resets/stops and there is check inside timer_set_config_state which prevents a running timer from being reset/stopped by not it's own timer_cb. Therefore timer_set_config_state returns -1 due to which rte_timer_reset returns -1 and rte_timer_reset_sync goes into an infinite loop. The soloution to this problem is to return -1 from rte_timer_reset_sync/rte_timer_stop_sync in case the user tries to reset/stop some other timer in it's callback function. Bugzilla ID: 491 Fixes: 20d159f20543 ("timer: fix corruption with reset") Cc: h.mikita89@gmail.com Signed-off-by: Sarosh Arif <sarosh.arif@emumba.com> --- v2: remove line continuations --- lib/librte_timer/rte_timer.c | 25 +++++++++++++++++++++++-- lib/librte_timer/rte_timer.h | 4 ++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/lib/librte_timer/rte_timer.c b/lib/librte_timer/rte_timer.c index 6d19ce469..be86385a1 100644 --- a/lib/librte_timer/rte_timer.c +++ b/lib/librte_timer/rte_timer.c @@ -576,14 +576,24 @@ rte_timer_alt_reset(uint32_t timer_data_id, struct rte_timer *tim, } /* loop until rte_timer_reset() succeed */ -void +int rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks, enum rte_timer_type type, unsigned tim_lcore, rte_timer_cb_t fct, void *arg) { + struct rte_timer_data *timer_data; + TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data, -EINVAL); + + if (tim->status.state == RTE_TIMER_RUNNING && + (tim->status.owner != (uint16_t)tim_lcore || + tim != timer_data->priv_timer[tim_lcore].running_tim)) + return -1; + while (rte_timer_reset(tim, ticks, type, tim_lcore, fct, arg) != 0) rte_pause(); + + return 0; } static int @@ -642,11 +652,22 @@ rte_timer_alt_stop(uint32_t timer_data_id, struct rte_timer *tim) } /* loop until rte_timer_stop() succeed */ -void +int rte_timer_stop_sync(struct rte_timer *tim) { + struct rte_timer_data *timer_data; + TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data, -EINVAL); + unsigned int lcore_id = rte_lcore_id(); + + if (tim->status.state == RTE_TIMER_RUNNING && + (tim->status.owner != (uint16_t)lcore_id || + tim != timer_data->priv_timer[lcore_id].running_tim)) + return -1; + while (rte_timer_stop(tim) != 0) rte_pause(); + + return 0; } /* Test the PENDING status of the timer handle tim */ diff --git a/lib/librte_timer/rte_timer.h b/lib/librte_timer/rte_timer.h index c6b3d450d..392ca423d 100644 --- a/lib/librte_timer/rte_timer.h +++ b/lib/librte_timer/rte_timer.h @@ -275,7 +275,7 @@ int rte_timer_reset(struct rte_timer *tim, uint64_t ticks, * @param arg * The user argument of the callback function. */ -void +int rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks, enum rte_timer_type type, unsigned tim_lcore, rte_timer_cb_t fct, void *arg); @@ -314,7 +314,7 @@ int rte_timer_stop(struct rte_timer *tim); * @param tim * The timer handle. */ -void rte_timer_stop_sync(struct rte_timer *tim); +int rte_timer_stop_sync(struct rte_timer *tim); /** * Test if a timer is pending. -- 2.17.1
On Wed, 8 Jul 2020 10:06:26 +0500
Sarosh Arif <sarosh.arif@emumba.com> wrote:
> /* loop until rte_timer_reset() succeed */
> -void
> +int
> rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks,
> enum rte_timer_type type, unsigned tim_lcore,
> rte_timer_cb_t fct, void *arg)
This is an API change and needs to wait until a breaking release like 20.11.
Also most applications won't test the result.
On Wed, 8 Jul 2020 10:06:26 +0500
Sarosh Arif <sarosh.arif@emumba.com> wrote:
> rte_timer_stop_sync(struct rte_timer *tim)
> {
> + struct rte_timer_data *timer_data;
> + TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data, -EINVAL);
> + unsigned int lcore_id = rte_lcore_id();
This mixing code and declarations. since the macro has a return statement.
Maybe:
struct rte_timer_data *timer_data;
unsigned int lcore_id = rte_lcore_id();
TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data, -EINVAL);
On Wed, Jul 8, 2020 at 8:08 PM Stephen Hemminger <stephen@networkplumber.org>
wrote:
> On Wed, 8 Jul 2020 10:06:26 +0500
> Sarosh Arif <sarosh.arif@emumba.com> wrote:
>
> > rte_timer_stop_sync(struct rte_timer *tim)
> > {
> > + struct rte_timer_data *timer_data;
> > + TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data,
> -EINVAL);
> > + unsigned int lcore_id = rte_lcore_id();
>
> This mixing code and declarations. since the macro has a return statement.
>
> Maybe:
>
> struct rte_timer_data *timer_data;
> unsigned int lcore_id = rte_lcore_id();
>
> TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data,
> -EINVAL);
>
I will fix this in the next version.
On Wed, Jul 8, 2020 at 8:07 PM Stephen Hemminger <stephen@networkplumber.org> wrote: > On Wed, 8 Jul 2020 10:06:26 +0500 > Sarosh Arif <sarosh.arif@emumba.com> wrote: > > > /* loop until rte_timer_reset() succeed */ > > -void > > +int > > rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks, > > enum rte_timer_type type, unsigned tim_lcore, > > rte_timer_cb_t fct, void *arg) > > This is an API change and needs to wait until a breaking release like > 20.11. > Okay,waiting till the next breaking release sounds fine to me. > Also most applications won't test the result. >
If the user tries to reset/stop some other timer in it's callback function, which is also about to expire, using rte_timer_reset_sync/rte_timer_stop_sync the application goes into an infinite loop. This happens because rte_timer_reset_sync/rte_timer_stop_sync loop until the timer resets/stops and there is check inside timer_set_config_state which prevents a running timer from being reset/stopped by not it's own timer_cb. Therefore timer_set_config_state returns -1 due to which rte_timer_reset returns -1 and rte_timer_reset_sync goes into an infinite loop. The soloution to this problem is to return -1 from rte_timer_reset_sync/rte_timer_stop_sync in case the user tries to reset/stop some other timer in it's callback function. Bugzilla ID: 491 Fixes: 20d159f20543 ("timer: fix corruption with reset") Cc: h.mikita89@gmail.com Signed-off-by: Sarosh Arif <sarosh.arif@emumba.com> --- v2: remove line continuations v3: separate code and declarations --- lib/librte_timer/rte_timer.c | 26 ++++++++++++++++++++++++-- lib/librte_timer/rte_timer.h | 4 ++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/lib/librte_timer/rte_timer.c b/lib/librte_timer/rte_timer.c index 6d19ce469..0cd3e2c86 100644 --- a/lib/librte_timer/rte_timer.c +++ b/lib/librte_timer/rte_timer.c @@ -576,14 +576,24 @@ rte_timer_alt_reset(uint32_t timer_data_id, struct rte_timer *tim, } /* loop until rte_timer_reset() succeed */ -void +int rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks, enum rte_timer_type type, unsigned tim_lcore, rte_timer_cb_t fct, void *arg) { + struct rte_timer_data *timer_data; + TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data, -EINVAL); + + if (tim->status.state == RTE_TIMER_RUNNING && + (tim->status.owner != (uint16_t)tim_lcore || + tim != timer_data->priv_timer[tim_lcore].running_tim)) + return -1; + while (rte_timer_reset(tim, ticks, type, tim_lcore, fct, arg) != 0) rte_pause(); + + return 0; } static int @@ -642,11 +652,23 @@ rte_timer_alt_stop(uint32_t timer_data_id, struct rte_timer *tim) } /* loop until rte_timer_stop() succeed */ -void +int rte_timer_stop_sync(struct rte_timer *tim) { + struct rte_timer_data *timer_data; + unsigned int lcore_id = rte_lcore_id(); + + TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data, -EINVAL); + + if (tim->status.state == RTE_TIMER_RUNNING && + (tim->status.owner != (uint16_t)lcore_id || + tim != timer_data->priv_timer[lcore_id].running_tim)) + return -1; + while (rte_timer_stop(tim) != 0) rte_pause(); + + return 0; } /* Test the PENDING status of the timer handle tim */ diff --git a/lib/librte_timer/rte_timer.h b/lib/librte_timer/rte_timer.h index c6b3d450d..392ca423d 100644 --- a/lib/librte_timer/rte_timer.h +++ b/lib/librte_timer/rte_timer.h @@ -275,7 +275,7 @@ int rte_timer_reset(struct rte_timer *tim, uint64_t ticks, * @param arg * The user argument of the callback function. */ -void +int rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks, enum rte_timer_type type, unsigned tim_lcore, rte_timer_cb_t fct, void *arg); @@ -314,7 +314,7 @@ int rte_timer_stop(struct rte_timer *tim); * @param tim * The timer handle. */ -void rte_timer_stop_sync(struct rte_timer *tim); +int rte_timer_stop_sync(struct rte_timer *tim); /** * Test if a timer is pending. -- 2.17.1
On Fri, 10 Jul 2020 11:59:54 +0500
Sarosh Arif <sarosh.arif@emumba.com> wrote:
> If the user tries to reset/stop some other timer in it's callback
> function, which is also about to expire, using
> rte_timer_reset_sync/rte_timer_stop_sync the application goes into
> an infinite loop. This happens because
> rte_timer_reset_sync/rte_timer_stop_sync loop until the timer
> resets/stops and there is check inside timer_set_config_state which
> prevents a running timer from being reset/stopped by not it's own
> timer_cb. Therefore timer_set_config_state returns -1 due to which
> rte_timer_reset returns -1 and rte_timer_reset_sync goes into an
> infinite loop.
>
> The soloution to this problem is to return -1 from
> rte_timer_reset_sync/rte_timer_stop_sync in case the user tries to
> reset/stop some other timer in it's callback function.
>
> Bugzilla ID: 491
> Fixes: 20d159f20543 ("timer: fix corruption with reset")
> Cc: h.mikita89@gmail.com
> Signed-off-by: Sarosh Arif <sarosh.arif@emumba.com>
> ---
> v2: remove line continuations
> v3: separate code and declarations
If you want to change the return value, you need to go through the steps
in the API/ABI policy. Maybe even symbol versioning.
Sorry, I know it is painful but we committed to the rules.
And changing the return value can never go to stable.
Hi Sarosh, Some comments in-line: > -----Original Message----- > From: Sarosh Arif <sarosh.arif@emumba.com> > Sent: Friday, July 10, 2020 2:00 AM > To: rsanford@akamai.com; Carrillo, Erik G <erik.g.carrillo@intel.com>; > dev@dpdk.org > Cc: stable@dpdk.org; Sarosh Arif <sarosh.arif@emumba.com>; > h.mikita89@gmail.com > Subject: [PATCH v3] lib/librte_timer:fix corruption with reset The subject is misleading - perhaps wording closer to the title of the Bugzilla bug would be more helpful. > > If the user tries to reset/stop some other timer in it's callback function, which > is also about to expire, using rte_timer_reset_sync/rte_timer_stop_sync the > application goes into an infinite loop. This happens because > rte_timer_reset_sync/rte_timer_stop_sync loop until the timer resets/stops > and there is check inside timer_set_config_state which prevents a running > timer from being reset/stopped by not it's own timer_cb. Therefore > timer_set_config_state returns -1 due to which rte_timer_reset returns -1 > and rte_timer_reset_sync goes into an infinite loop. > > The soloution to this problem is to return -1 from > rte_timer_reset_sync/rte_timer_stop_sync in case the user tries to > reset/stop some other timer in it's callback function. > > Bugzilla ID: 491 > Fixes: 20d159f20543 ("timer: fix corruption with reset") > Cc: h.mikita89@gmail.com > Signed-off-by: Sarosh Arif <sarosh.arif@emumba.com> > --- > v2: remove line continuations > v3: separate code and declarations > --- > lib/librte_timer/rte_timer.c | 26 ++++++++++++++++++++++++-- > lib/librte_timer/rte_timer.h | 4 ++-- > 2 files changed, 26 insertions(+), 4 deletions(-) > > diff --git a/lib/librte_timer/rte_timer.c b/lib/librte_timer/rte_timer.c index > 6d19ce469..0cd3e2c86 100644 > --- a/lib/librte_timer/rte_timer.c > +++ b/lib/librte_timer/rte_timer.c > @@ -576,14 +576,24 @@ rte_timer_alt_reset(uint32_t timer_data_id, struct > rte_timer *tim, } > > /* loop until rte_timer_reset() succeed */ -void > +int > rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks, > enum rte_timer_type type, unsigned tim_lcore, > rte_timer_cb_t fct, void *arg) > { > + struct rte_timer_data *timer_data; > + TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, > timer_data, -EINVAL); > + > + if (tim->status.state == RTE_TIMER_RUNNING && > + (tim->status.owner != (uint16_t)tim_lcore || > + tim != timer_data->priv_timer[tim_lcore].running_tim)) > + return -1; > + As I understand it, Bugzilla 491 describes two scenarios where a hang can occur: 1. A timer's callback tries to synchronously reset/stop another timer in the same run list 2. A timer's callback tries to synchronously reset/stop another timer in a different run list whose lcore happens to be running a timer callback that is synchronously resetting/stopping a timer in the first run list The if condition from the patch above can be broken up as: (tim->status.state == RTE_TIMER_RUNNING && tim->status.owner == (uint16_t)lcore_id && tim != timer_data->priv_timer[lcore_id].running_tim) And (tim->status.state == RTE_TIMER_RUNNING && tim->status.owner != (uint16_t)lcore_id) This second condition could be transient and doesn't necessarily identify scenario (2) above. In this case, the *_sync() calls could fail unnecessarily. Offhand, I'm not seeing a way to more precisely detect scenario 2 above. I'm wondering if some kind of a timeout parameter could be added to avoid hanging instead. Thoughts? As Stephen mentioned in another response, it looks like this will require an API change. I believe this can be announced in the next release via doc/guides/rel_notes/deprecation.rst. Then, the new API can be added in the next ABI-breaking release, possibly with versioned symbols (http://doc.dpdk.org/guides/contributing/abi_versioning.html#versioning-macros). Thanks, Erik > while (rte_timer_reset(tim, ticks, type, tim_lcore, > fct, arg) != 0) > rte_pause(); > + > + return 0; > } > > static int > @@ -642,11 +652,23 @@ rte_timer_alt_stop(uint32_t timer_data_id, struct > rte_timer *tim) } > > /* loop until rte_timer_stop() succeed */ -void > +int > rte_timer_stop_sync(struct rte_timer *tim) { > + struct rte_timer_data *timer_data; > + unsigned int lcore_id = rte_lcore_id(); > + > + TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, > timer_data, -EINVAL); > + > + if (tim->status.state == RTE_TIMER_RUNNING && > + (tim->status.owner != (uint16_t)lcore_id || > + tim != timer_data->priv_timer[lcore_id].running_tim)) > + return -1; > + > while (rte_timer_stop(tim) != 0) > rte_pause(); > + > + return 0; > } > > /* Test the PENDING status of the timer handle tim */ diff --git > a/lib/librte_timer/rte_timer.h b/lib/librte_timer/rte_timer.h index > c6b3d450d..392ca423d 100644 > --- a/lib/librte_timer/rte_timer.h > +++ b/lib/librte_timer/rte_timer.h > @@ -275,7 +275,7 @@ int rte_timer_reset(struct rte_timer *tim, uint64_t > ticks, > * @param arg > * The user argument of the callback function. > */ > -void > +int > rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks, > enum rte_timer_type type, unsigned tim_lcore, > rte_timer_cb_t fct, void *arg); > @@ -314,7 +314,7 @@ int rte_timer_stop(struct rte_timer *tim); > * @param tim > * The timer handle. > */ > -void rte_timer_stop_sync(struct rte_timer *tim); > +int rte_timer_stop_sync(struct rte_timer *tim); > > /** > * Test if a timer is pending. > -- > 2.17.1