Re: [PATCH v6 1/3] eal: add rte thread create control API

[David Marchand <david.marchand@redhat.com>]

On Wed, Feb 8, 2023 at 10:26 PM Tyler Retzlaff
<roretzla@linux.microsoft.com> wrote:
>
> Add rte_thread_create_control API as a replacement for
> rte_ctrl_thread_create to allow deprecation of the use of platform
> specific types in DPDK public API.
>
> Add test from David Marchand to exercise the new API.
>
> Signed-off-by: Tyler Retzlaff <roretzla@linux.microsoft.com>
> Acked-by: Morten Brørup <mb@smartsharesystems.com>
> Reviewed-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> ---
>  app/test/test_threads.c            | 26 ++++++++++++
>  lib/eal/common/eal_common_thread.c | 85 ++++++++++++++++++++++++++++++++++----
>  lib/eal/include/rte_thread.h       | 33 +++++++++++++++
>  lib/eal/version.map                |  1 +
>  4 files changed, 137 insertions(+), 8 deletions(-)
>
> diff --git a/app/test/test_threads.c b/app/test/test_threads.c
> index e0f18e4..657ecad 100644
> --- a/app/test/test_threads.c
> +++ b/app/test/test_threads.c
> @@ -232,6 +232,31 @@
>         return 0;
>  }
>
> +static int
> +test_thread_create_control_join(void)
> +{
> +       rte_thread_t thread_id;
> +       rte_thread_t thread_main_id;
> +
> +       thread_id_ready = 0;
> +       RTE_TEST_ASSERT(rte_thread_create_control(&thread_id, "test_control_threads",
> +               NULL, thread_main, &thread_main_id) == 0,
> +               "Failed to create thread.");
> +
> +       while (__atomic_load_n(&thread_id_ready, __ATOMIC_ACQUIRE) == 0)
> +               ;
> +
> +       RTE_TEST_ASSERT(rte_thread_equal(thread_id, thread_main_id) != 0,
> +               "Unexpected thread id.");
> +
> +       __atomic_store_n(&thread_id_ready, 2, __ATOMIC_RELEASE);
> +
> +       RTE_TEST_ASSERT(rte_thread_join(thread_id, NULL) == 0,
> +               "Failed to join thread.");
> +
> +       return 0;
> +}
> +
>  static struct unit_test_suite threads_test_suite = {
>         .suite_name = "threads autotest",
>         .setup = NULL,
> @@ -243,6 +268,7 @@
>                 TEST_CASE(test_thread_priority),
>                 TEST_CASE(test_thread_attributes_affinity),
>                 TEST_CASE(test_thread_attributes_priority),
> +               TEST_CASE(test_thread_create_control_join),
>                 TEST_CASES_END()
>         }
>  };
> diff --git a/lib/eal/common/eal_common_thread.c b/lib/eal/common/eal_common_thread.c
> index 3181515..4f83c97 100644
> --- a/lib/eal/common/eal_common_thread.c
> +++ b/lib/eal/common/eal_common_thread.c
> @@ -232,7 +232,10 @@ enum __rte_ctrl_thread_status {
>  };
>
>  struct rte_thread_ctrl_params {
> -       void *(*start_routine)(void *);
> +       union {
> +               void *(*ctrl_start_routine)(void *arg);
> +               rte_thread_func control_start_routine;
> +       } u;
>         void *arg;
>         int ret;
>         /* Control thread status.
> @@ -241,27 +244,47 @@ struct rte_thread_ctrl_params {
>         enum __rte_ctrl_thread_status ctrl_thread_status;
>  };
>
> -static void *ctrl_thread_init(void *arg)
> +static int ctrl_thread_init(void *arg)
>  {
>         struct internal_config *internal_conf =
>                 eal_get_internal_configuration();
>         rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset;
>         struct rte_thread_ctrl_params *params = arg;
> -       void *(*start_routine)(void *) = params->start_routine;
> -       void *routine_arg = params->arg;
>
>         __rte_thread_init(rte_lcore_id(), cpuset);
>         params->ret = rte_thread_set_affinity_by_id(rte_thread_self(), cpuset);
>         if (params->ret != 0) {
>                 __atomic_store_n(&params->ctrl_thread_status,
>                         CTRL_THREAD_ERROR, __ATOMIC_RELEASE);
> -               return NULL;
> +               return params->ret;
>         }
>
>         __atomic_store_n(&params->ctrl_thread_status,
>                 CTRL_THREAD_RUNNING, __ATOMIC_RELEASE);
>
> -       return start_routine(routine_arg);
> +       return 0;
> +}
> +
> +static void *ctrl_thread_start(void *arg)
> +{
> +       struct rte_thread_ctrl_params *params = arg;
> +       void *(*start_routine)(void *) = params->u.ctrl_start_routine;
> +
> +       if (ctrl_thread_init(arg) != 0)
> +               return NULL;
> +
> +       return start_routine(params->arg);

Does this patch open a race on params->arg between the newly created
thread, and the one that asked for creation?
Reading params->arg is not under the ctrl_thread_status
syncrhonisation point anymore (which was protecting against
free(params)).

WDYT?

(Note that I doubt this patch has anything to do with the recent bz
opened by Intel QE)


> +}
> +
> +static uint32_t control_thread_start(void *arg)
> +{
> +       struct rte_thread_ctrl_params *params = arg;
> +       rte_thread_func start_routine = params->u.control_start_routine;
> +
> +       if (ctrl_thread_init(arg) != 0)
> +               return params->ret;

Idem here for params->ret.


> +
> +       return start_routine(params->arg);
>  }
>
>  int
> @@ -277,12 +300,12 @@ static void *ctrl_thread_init(void *arg)
>         if (!params)
>                 return -ENOMEM;
>
> -       params->start_routine = start_routine;
> +       params->u.ctrl_start_routine = start_routine;
>         params->arg = arg;
>         params->ret = 0;
>         params->ctrl_thread_status = CTRL_THREAD_LAUNCHING;
>
> -       ret = pthread_create(thread, attr, ctrl_thread_init, (void *)params);
> +       ret = pthread_create(thread, attr, ctrl_thread_start, (void *)params);
>         if (ret != 0) {
>                 free(params);
>                 return -ret;
> @@ -315,6 +338,52 @@ static void *ctrl_thread_init(void *arg)
>  }
>
>  int
> +rte_thread_create_control(rte_thread_t *thread, const char *name,
> +       const rte_thread_attr_t *attr,
> +       rte_thread_func start_routine, void *arg)
> +{
> +       struct rte_thread_ctrl_params *params;
> +       enum __rte_ctrl_thread_status ctrl_thread_status;
> +       int ret;
> +
> +       params = malloc(sizeof(*params));
> +       if (params == NULL)
> +               return -ENOMEM;
> +
> +       params->u.control_start_routine = start_routine;
> +       params->arg = arg;
> +       params->ret = 0;
> +       params->ctrl_thread_status = CTRL_THREAD_LAUNCHING;
> +
> +       ret = rte_thread_create(thread, attr, control_thread_start, params);
> +       if (ret != 0) {
> +               free(params);
> +               return -ret;
> +       }
> +
> +       if (name != NULL)
> +               rte_thread_set_name(*thread, name);
> +
> +       /* Wait for the control thread to initialize successfully */
> +       while ((ctrl_thread_status =
> +                       __atomic_load_n(&params->ctrl_thread_status,
> +                       __ATOMIC_ACQUIRE)) == CTRL_THREAD_LAUNCHING) {
> +               rte_delay_us_sleep(1);
> +       }
> +
> +       /* Check if the control thread encountered an error */
> +       if (ctrl_thread_status == CTRL_THREAD_ERROR) {
> +               /* ctrl thread is exiting */
> +               rte_thread_join(*thread, NULL);
> +       }
> +
> +       ret = params->ret;
> +       free(params);
> +
> +       return ret;
> +}
> +
> +int
>  rte_thread_register(void)
>  {
>         unsigned int lcore_id;
> diff --git a/lib/eal/include/rte_thread.h b/lib/eal/include/rte_thread.h
> index d247930..fae26a7 100644
> --- a/lib/eal/include/rte_thread.h
> +++ b/lib/eal/include/rte_thread.h
> @@ -98,6 +98,39 @@ int rte_thread_create(rte_thread_t *thread_id,
>   * @warning
>   * @b EXPERIMENTAL: this API may change without prior notice.
>   *
> + * Create a control thread.
> + *
> + * Creates a control thread with the given name and attributes. The
> + * affinity of the new thread is based on the CPU affinity retrieved
> + * at the time rte_eal_init() was called, the EAL threads are then
> + * excluded. If setting the name of the thread fails, the error is
> + * ignored and a debug message is logged.
> + *
> + * @param thread
> + *   Filled with the thread id of the new created thread.
> + * @param name
> + *   The name of the control thread
> + *   (max RTE_MAX_THREAD_NAME_LEN characters including '\0').
> + * @param thread_attr
> + *   Attributes for the new thread.
> + * @param thread_func
> + *   Function to be executed by the new thread.
> + * @param arg
> + *   Argument passed to start_routine.
> + * @return
> + *   On success, returns 0; on error, it returns a negative value
> + *   corresponding to the error number.
> + */
> +__rte_experimental
> +int
> +rte_thread_create_control(rte_thread_t *thread, const char *name,
> +               const rte_thread_attr_t *thread_attr,
> +               rte_thread_func thread_func, void *arg);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
>   * Waits for the thread identified by 'thread_id' to terminate
>   *
>   * @param thread_id
> diff --git a/lib/eal/version.map b/lib/eal/version.map
> index 6523102..285a0bd 100644
> --- a/lib/eal/version.map
> +++ b/lib/eal/version.map
> @@ -441,6 +441,7 @@ EXPERIMENTAL {
>         rte_thread_join;
>
>         # added in 23.03
> +       rte_thread_create_control;
>         rte_thread_set_name;
>  };
>
> --
> 1.8.3.1
>


-- 
David Marchand