Re: [PATCH v7 7/8] trace: add PMU

[David Marchand <david.marchand@redhat.com>]

On Fri, Jun 27, 2025 at 5:41 PM Tomasz Duszynski <tduszynski@marvell.com> wrote:
>
> In order to profile app, one needs to store significant amount of samples
> somewhere for an analysis later on.
> Since trace library supports storing data in a CTF format,
> lets take advantage of that and add a dedicated PMU tracepoint.
>
> Signed-off-by: Tomasz Duszynski <tduszynski@marvell.com>
> ---
>  MAINTAINERS                                |  1 +
>  app/test/test_trace_perf.c                 | 10 ++++
>  doc/guides/prog_guide/profile_app.rst      |  5 ++
>  doc/guides/prog_guide/trace_lib.rst        | 31 ++++++++++
>  doc/guides/rel_notes/release_25_07.rst     |  2 +
>  lib/eal/common/eal_common_trace.c          |  5 +-
>  lib/eal/common/eal_common_trace_pmu.c      | 38 ++++++++++++
>  lib/eal/common/eal_common_trace_points.c   | 20 +++++++
>  lib/eal/common/eal_trace.h                 |  4 ++
>  lib/eal/common/meson.build                 |  1 +
>  lib/eal/include/rte_eal_trace.h            | 16 +++++
>  lib/eal/include/rte_trace_point.h          |  7 +++
>  lib/eal/include/rte_trace_point_register.h |  2 +
>  lib/eal/meson.build                        |  3 +
>  lib/meson.build                            |  2 +-
>  lib/pmu/pmu.c                              | 69 +++++++++++++++++++++-
>  lib/pmu/rte_pmu.h                          | 24 ++++++++
>  17 files changed, 236 insertions(+), 4 deletions(-)
>  create mode 100644 lib/eal/common/eal_common_trace_pmu.c
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 0e9357f3a3..74cc8fc195 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -1850,6 +1850,7 @@ F: doc/guides/prog_guide/eventdev/dispatcher_lib.rst
>  PMU - EXPERIMENTAL
>  M: Tomasz Duszynski <tduszynski@marvell.com>
>  F: lib/pmu/
> +F: lib/eal/common/eal_common_trace_pmu.c
>  F: app/test/test_pmu.c
>
>  Job statistics
> diff --git a/app/test/test_trace_perf.c b/app/test/test_trace_perf.c
> index 8257cc02be..28f908ce40 100644
> --- a/app/test/test_trace_perf.c
> +++ b/app/test/test_trace_perf.c
> @@ -114,6 +114,10 @@ worker_fn_##func(void *arg) \
>  #define GENERIC_DOUBLE rte_eal_trace_generic_double(3.66666)
>  #define GENERIC_STR rte_eal_trace_generic_str("hello world")
>  #define VOID_FP app_dpdk_test_fp()
> +#ifdef RTE_LIB_PMU
> +/* 0 corresponds first event passed via --trace= */
> +#define READ_PMU rte_pmu_trace_read(0)
> +#endif
>
>  WORKER_DEFINE(GENERIC_VOID)
>  WORKER_DEFINE(GENERIC_U64)
> @@ -122,6 +126,9 @@ WORKER_DEFINE(GENERIC_FLOAT)
>  WORKER_DEFINE(GENERIC_DOUBLE)
>  WORKER_DEFINE(GENERIC_STR)
>  WORKER_DEFINE(VOID_FP)
> +#ifdef RTE_LIB_PMU
> +WORKER_DEFINE(READ_PMU)
> +#endif
>
>  static void
>  run_test(const char *str, lcore_function_t f, struct test_data *data, size_t sz)
> @@ -174,6 +181,9 @@ test_trace_perf(void)
>         run_test("double", worker_fn_GENERIC_DOUBLE, data, sz);
>         run_test("string", worker_fn_GENERIC_STR, data, sz);
>         run_test("void_fp", worker_fn_VOID_FP, data, sz);
> +#ifdef RTE_LIB_PMU
> +       run_test("read_pmu", worker_fn_READ_PMU, data, sz);
> +#endif
>
>         rte_free(data);
>         return TEST_SUCCESS;
> diff --git a/doc/guides/prog_guide/profile_app.rst b/doc/guides/prog_guide/profile_app.rst
> index 2f47680d5d..362fd20143 100644
> --- a/doc/guides/prog_guide/profile_app.rst
> +++ b/doc/guides/prog_guide/profile_app.rst
> @@ -42,6 +42,11 @@ Current implementation imposes certain limitations:
>  * EAL lcores must not share a CPU.
>  * Each EAL lcore measures the same group of events.
>
> +Alternatively tracing library can be used,
> +which offers dedicated tracepoint ``rte_pmu_trace_read()``.
> +
> +Refer to :doc:`../prog_guide/trace_lib` for more details.
> +
>
>  Profiling on x86
>  ----------------
> diff --git a/doc/guides/prog_guide/trace_lib.rst b/doc/guides/prog_guide/trace_lib.rst
> index d9b17abe90..97158cce37 100644
> --- a/doc/guides/prog_guide/trace_lib.rst
> +++ b/doc/guides/prog_guide/trace_lib.rst
> @@ -46,6 +46,7 @@ DPDK tracing library features
>    trace format and is compatible with ``LTTng``.
>    For detailed information, refer to
>    `Common Trace Format <https://diamon.org/ctf/>`_.
> +- Support reading PMU events on ARM64 and x86-64 (Intel)
>
>  How to add a tracepoint?
>  ------------------------
> @@ -139,6 +140,36 @@ the user must use ``RTE_TRACE_POINT_FP`` instead of ``RTE_TRACE_POINT``.
>  ``RTE_TRACE_POINT_FP`` is compiled out by default and it can be enabled using
>  the ``enable_trace_fp`` option for meson build.
>
> +PMU tracepoint
> +--------------
> +
> +Performance Monitoring Unit (PMU) event values can be read from hardware registers
> +using the predefined ``rte_pmu_read`` tracepoint.
> +
> +Tracing is enabled via ``--trace`` EAL option by passing both expression
> +matching PMU tracepoint name i.e ``lib.eal.pmu.read``
> +and expression ``e=ev1[,ev2,...]`` matching particular events::
> +
> +    --trace='.*pmu.read\|e=cpu_cycles,l1d_cache'
> +
> +Event names are available under ``/sys/bus/event_source/devices/PMU/events`` directory,
> +where ``PMU`` is a placeholder for either a ``cpu`` or a directory containing ``cpus``.
> +
> +In contrary to other tracepoints this does not need any extra variables
> +added to source files.
> +Instead, caller passes index
> +which follows the order of events specified via ``--trace`` parameter.
> +In the following example, index ``0`` corresponds to ``cpu_cyclces``,
> +while index ``1`` corresponds to ``l1d_cache``.
> +
> +.. code-block:: c
> +
> +   rte_pmu_trace_read(0);
> +   rte_pmu_trace_read(1);
> +
> +PMU tracing support must be explicitly enabled
> +using the ``enable_trace_fp`` option for Meson build.
> +
>  Event record mode
>  -----------------
>
> diff --git a/doc/guides/rel_notes/release_25_07.rst b/doc/guides/rel_notes/release_25_07.rst
> index 19be7740c4..3c55f93853 100644
> --- a/doc/guides/rel_notes/release_25_07.rst
> +++ b/doc/guides/rel_notes/release_25_07.rst
> @@ -71,6 +71,8 @@ New Features
>
>    Added a Performance Monitoring Unit (PMU) library which allows Linux applications
>    to perform self monitoring activities without depending on external utilities like perf.
> +  After integration with :doc:`../prog_guide/trace_lib`, data gathered from hardware counters
> +  can be stored in CTF format for further analysis.
>
>  * **Added Mucse rnp net driver.**
>
> diff --git a/lib/eal/common/eal_common_trace.c b/lib/eal/common/eal_common_trace.c
> index be1f78a68d..45e7f9aa56 100644
> --- a/lib/eal/common/eal_common_trace.c
> +++ b/lib/eal/common/eal_common_trace.c
> @@ -75,8 +75,10 @@ eal_trace_init(void)
>                 goto free_meta;
>
>         /* Apply global configurations */
> -       STAILQ_FOREACH(arg, &trace.args, next)
> +       STAILQ_FOREACH(arg, &trace.args, next) {
>                 trace_args_apply(arg->val);
> +               trace_pmu_args_apply(arg->val);
> +       }
>
>         rte_trace_mode_set(trace.mode);
>
> @@ -92,6 +94,7 @@ eal_trace_init(void)
>  void
>  eal_trace_fini(void)
>  {
> +       trace_pmu_args_free();
>         trace_mem_free();
>         trace_metadata_destroy();
>         eal_trace_args_free();
> diff --git a/lib/eal/common/eal_common_trace_pmu.c b/lib/eal/common/eal_common_trace_pmu.c
> new file mode 100644
> index 0000000000..3824904481
> --- /dev/null
> +++ b/lib/eal/common/eal_common_trace_pmu.c
> @@ -0,0 +1,38 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(C) 2025 Marvell International Ltd.
> + */
> +
> +#include <rte_common.h>
> +
> +#include "eal_trace.h"
> +
> +#ifdef RTE_LIB_PMU
> +
> +#include <rte_pmu.h>
> +
> +void
> +trace_pmu_args_apply(const char *arg)
> +{
> +       static bool once;
> +
> +       if (!once) {
> +               if (rte_pmu_init())
> +                       return;
> +               once = true;
> +       }
> +
> +       rte_pmu_add_events_by_pattern(arg);
> +}
> +
> +void
> +trace_pmu_args_free(void)
> +{
> +       rte_pmu_fini();
> +}
> +
> +#else /* !RTE_LIB_PMU */
> +
> +void trace_pmu_args_apply(const char *arg __rte_unused) { return; }
> +void trace_pmu_args_free(void) { return; }
> +
> +#endif /* RTE_LIB_PMU */
> diff --git a/lib/eal/common/eal_common_trace_points.c b/lib/eal/common/eal_common_trace_points.c
> index 0903f3c639..ea90279d77 100644
> --- a/lib/eal/common/eal_common_trace_points.c
> +++ b/lib/eal/common/eal_common_trace_points.c
> @@ -119,3 +119,23 @@ RTE_TRACE_POINT_REGISTER(rte_eal_trace_intr_enable,
>         lib.eal.intr.enable)
>  RTE_TRACE_POINT_REGISTER(rte_eal_trace_intr_disable,
>         lib.eal.intr.disable)
> +
> +#ifdef RTE_LIB_PMU
> +RTE_EXPORT_EXPERIMENTAL_SYMBOL(__rte_pmu_trace_read, 25.07)
> +RTE_TRACE_POINT_REGISTER(rte_pmu_trace_read,
> +       lib.pmu.read)
> +#endif
> +#ifdef RTE_EXEC_ENV_IS_WINDOWS
> +/* gen-version-map.py script generates export symbol maps by scanning source files without
> + * evaluating conditional compilation. Hence __rte_pmu_trace_read will be included the version map
> + * even if library is not compiled.
> + *
> + * On Windows if msvc linker is used this leads to a hard link error
> + * (LNK2001: unresolved external symbol) because msvc requires all symbols listed in the .def file
> + * to be present in the object files.
> + *
> + * Other linkers, e.g: gnu ld or mingw ld, are more forgiving. They silently ignore symbols listed
> + * in the map file if those symbols are not present in the binary.
> + */
> +rte_trace_point_t __rte_pmu_trace_read;
> +#endif

From a quick look, could you export this symbol from the PMU library itself?


-- 
David Marchand