[PATCH v2 0/3] add uncore api to be called through l3fwd-power

[PATCH v2 0/3] add uncore api to be called through l3fwd-power

[Tadhg Kearney]

This is targeting 22.11 and aims to add an API to DPDK to allow uncore frequency adjustment.
This will be called through the l3fwd-power app, and gives the ability to set the min,
max and specific frequency index that you want the uncore to be at.

Signed-off-by: Tadhg Kearney <tadhg.kearney@intel.com>
---

v2:
Fix compilation warnings and errors.

Tadhg Kearney (3):
  power: add uncore API to power library
  test/power: add unit tests for uncore API
  l3fwd-power: add option to call uncore API

 app/test/meson.build                          |   2 +
 app/test/test_power_uncore.c                  | 245 +++++++++++
 config/x86/meson.build                        |   2 +
 doc/guides/prog_guide/power_man.rst           |  27 ++
 .../sample_app_ug/l3_forward_power_man.rst    |  28 ++
 examples/l3fwd-power/main.c                   | 190 ++++++++-
 lib/power/meson.build                         |   2 +
 lib/power/rte_power_uncore.c                  | 399 ++++++++++++++++++
 lib/power/rte_power_uncore.h                  | 159 +++++++
 lib/power/version.map                         |   7 +
 10 files changed, 1045 insertions(+), 16 deletions(-)
 create mode 100644 app/test/test_power_uncore.c
 create mode 100644 lib/power/rte_power_uncore.c
 create mode 100644 lib/power/rte_power_uncore.h

-- 
2.25.1

[PATCH v2 1/3] power: add uncore API to power library

[Tadhg Kearney]

Add api to allow uncore frequency adjustment. This is done through
manipulating related sysfs entries to adjust the min/max uncore values.
Seven api's are being added that are all public and experimental.

Signed-off-by: Tadhg Kearney <tadhg.kearney@intel.com>
---
 config/x86/meson.build              |   2 +
 doc/guides/prog_guide/power_man.rst |  27 ++
 lib/power/meson.build               |   2 +
 lib/power/rte_power_uncore.c        | 399 ++++++++++++++++++++++++++++
 lib/power/rte_power_uncore.h        | 159 +++++++++++
 lib/power/version.map               |   7 +
 6 files changed, 596 insertions(+)
 create mode 100644 lib/power/rte_power_uncore.c
 create mode 100644 lib/power/rte_power_uncore.h

diff --git a/config/x86/meson.build b/config/x86/meson.build
index 54345c4da3..f81103d8e3 100644
--- a/config/x86/meson.build
+++ b/config/x86/meson.build
@@ -73,3 +73,5 @@ endif
 dpdk_conf.set('RTE_CACHE_LINE_SIZE', 64)
 dpdk_conf.set('RTE_MAX_LCORE', 128)
 dpdk_conf.set('RTE_MAX_NUMA_NODES', 32)
+dpdk_conf.set('RTE_MAX_NUMA_DIE', 1)
+dpdk_conf.set('RTE_MAX_UNCORE_FREQS', 32)
diff --git a/doc/guides/prog_guide/power_man.rst b/doc/guides/prog_guide/power_man.rst
index 98cfd3c1f3..ec975a5556 100644
--- a/doc/guides/prog_guide/power_man.rst
+++ b/doc/guides/prog_guide/power_man.rst
@@ -276,6 +276,33 @@ API Overview for Ethernet PMD Power Management
 * **Set Scaling Max Freq**: Set the maximum frequency (kHz) to be used in Frequency
   Scaling mode.
 
+Uncore API
+----------
+
+Abstract
+~~~~~~~~
+Up to 60% power saving can be achieved by reducing the uncore frequency to its lowest value.
+With later kernels, there is now a sysfs entry to allow adjustment of uncore frequency.
+This manipulates the contest of MSR 0x620, which sets min/max of the uncore for the SKU.
+
+
+API Overview for Uncore
+~~~~~~~~~~~~~~~~~~~~~~~
+* **Uncore Power Init**: Initialise uncore power, populate frequency array and record
+  original min & max for pkg & die.
+
+* **Uncore Power Exit**: Exit uncore power, restoring original min & max for pkg & die.
+
+* **Get Uncore Power Freq**: Get current uncore freq index for pkg & die.
+
+* **Set Uncore Power Freq**: Set min & max uncore freq index for pkg & die (min and max will be the same).
+
+* **Uncore Power Max**: Set max uncore freq index for pkg & die.
+
+* **Uncore Power Min**: Set min uncore freq index for pkg & die.
+
+* **Get Num Freqs**: Get the number of frequencies in the index array.
+
 References
 ----------
 
diff --git a/lib/power/meson.build b/lib/power/meson.build
index ba8d66074b..80cdeb72d4 100644
--- a/lib/power/meson.build
+++ b/lib/power/meson.build
@@ -21,12 +21,14 @@ sources = files(
         'rte_power.c',
         'rte_power_empty_poll.c',
         'rte_power_pmd_mgmt.c',
+        'rte_power_uncore.c',
 )
 headers = files(
         'rte_power.h',
         'rte_power_empty_poll.h',
         'rte_power_pmd_mgmt.h',
         'rte_power_guest_channel.h',
+        'rte_power_uncore.h',
 )
 if cc.has_argument('-Wno-cast-qual')
     cflags += '-Wno-cast-qual'
diff --git a/lib/power/rte_power_uncore.c b/lib/power/rte_power_uncore.c
new file mode 100644
index 0000000000..184df7013b
--- /dev/null
+++ b/lib/power/rte_power_uncore.c
@@ -0,0 +1,399 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2022 Intel Corporation
+ */
+
+#include <errno.h>
+
+#include <rte_memcpy.h>
+
+#include "rte_power_uncore.h"
+#include "power_common.h"
+
+#define BUS_FREQ     100000
+
+#define POWER_GOVERNOR_PERF "performance"
+#define POWER_UNCORE_SYSFILE_MAX_FREQ \
+		"/sys/devices/system/cpu/intel_uncore_frequency/package_%02u_die_%02u/max_freq_khz"
+#define POWER_UNCORE_SYSFILE_MIN_FREQ  \
+		"/sys/devices/system/cpu/intel_uncore_frequency/package_%02u_die_%02u/min_freq_khz"
+#define POWER_UNCORE_SYSFILE_BASE_MAX_FREQ \
+		"/sys/devices/system/cpu/intel_uncore_frequency/package_%02u_die_%02u/initial_max_freq_khz"
+#define POWER_UNCORE_SYSFILE_BASE_MIN_FREQ  \
+		"/sys/devices/system/cpu/intel_uncore_frequency/package_%02u_die_%02u/initial_min_freq_khz"
+
+
+struct uncore_power_info {
+	unsigned int die;                    /**< Core die id */
+	unsigned int pkg;                    /**< Package id */
+	uint32_t freqs[RTE_MAX_UNCORE_FREQS];/**< Frequency array */
+	uint32_t nb_freqs;                   /**< Number of available freqs */
+	FILE *f_cur_min;                     /**< FD of scaling_min */
+	FILE *f_cur_max;                     /**< FD of scaling_max */
+	FILE *f_base_min;                    /**< FD of initial min */
+	FILE *f_base_max;                    /**< FD of initial max */
+	int cur_idx;                         /**< Freq index in freqs array */
+	uint32_t init_max_freq;              /**< Initial max frequency */
+	uint32_t init_min_freq;              /**< Initial min frequency */
+} __rte_cache_aligned;
+
+
+static struct uncore_power_info uncore_info[RTE_MAX_NUMA_NODES][RTE_MAX_NUMA_DIE];
+
+static int
+set_uncore_freq_internal(struct uncore_power_info *ui, uint32_t idx)
+{
+	uint32_t target_uncore_freq, curr_max_freq;
+	int ret;
+
+	if (idx >= RTE_MAX_UNCORE_FREQS || idx >= ui->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Invalid uncore frequency index %u, which "
+				"should be less than %u\n", idx, ui->nb_freqs);
+		return -1;
+	}
+
+	target_uncore_freq = ui->freqs[idx];
+
+	if (fprintf(ui->f_cur_min, "%u", target_uncore_freq) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new uncore frequency for "
+				"pkg %02u die %02u\n", ui->pkg, ui->die);
+		return -1;
+	}
+
+	if (fprintf(ui->f_cur_max, "%u", target_uncore_freq) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new uncore frequency for "
+				"pkg %02u die %02u\n", ui->pkg, ui->die);
+		return -1;
+	}
+
+	POWER_DEBUG_TRACE("Uncore requency '%u' to be set for pkg %02u die %02u\n",
+				target_uncore_freq, ui->pkg, ui->die);
+
+	open_core_sysfs_file(&ui->f_cur_max, "rw+", POWER_UNCORE_SYSFILE_MAX_FREQ,
+			ui->pkg, ui->die);
+	if (ui->f_cur_max == NULL) {
+		RTE_LOG(DEBUG, POWER, "failed to open %s\n",
+				POWER_UNCORE_SYSFILE_MAX_FREQ);
+		return -1;
+	}
+	ret = read_core_sysfs_u32(ui->f_cur_max, &curr_max_freq);
+		if (ret < 0) {
+			RTE_LOG(DEBUG, POWER, "Failed to read %s\n",
+					POWER_UNCORE_SYSFILE_MAX_FREQ);
+			fclose(ui->f_cur_max);
+			return -1;
+		}
+
+	if (target_uncore_freq <= curr_max_freq) {
+		fflush(ui->f_cur_min);
+		fflush(ui->f_cur_max);
+	} else {
+		fflush(ui->f_cur_max);
+		fflush(ui->f_cur_min);
+	}
+
+	ui->cur_idx = idx;
+
+	return 0;
+}
+
+/**
+ * Fopen the sys file for the future setting of the uncore die frequency.
+ */
+static int
+power_init_for_setting_uncore_freq(struct uncore_power_info *ui)
+{
+	FILE *f_base_min = NULL, *f_base_max = NULL, *f_min = NULL, *f_max = NULL;
+	uint32_t base_min_freq, base_max_freq, min_freq, max_freq;
+	int ret;
+
+	/* open and read all uncore sys files */
+	/* Base_max */
+	open_core_sysfs_file(&f_base_max, "r", POWER_UNCORE_SYSFILE_BASE_MAX_FREQ,
+			ui->pkg, ui->die);
+	if (f_base_max == NULL) {
+		RTE_LOG(DEBUG, POWER, "failed to open %s\n",
+				POWER_UNCORE_SYSFILE_BASE_MAX_FREQ);
+		goto err;
+	}
+	ret = read_core_sysfs_u32(f_base_max, &base_max_freq);
+	if (ret < 0) {
+		RTE_LOG(DEBUG, POWER, "Failed to read %s\n",
+				POWER_UNCORE_SYSFILE_BASE_MAX_FREQ);
+		goto err;
+	}
+
+	/* Base min */
+	open_core_sysfs_file(&f_base_min, "r", POWER_UNCORE_SYSFILE_BASE_MIN_FREQ,
+		ui->pkg, ui->die);
+	if (f_base_min == NULL) {
+		RTE_LOG(DEBUG, POWER, "failed to open %s\n",
+				POWER_UNCORE_SYSFILE_BASE_MIN_FREQ);
+		goto err;
+	}
+	if (f_base_min != NULL) {
+		ret = read_core_sysfs_u32(f_base_min, &base_min_freq);
+		if (ret < 0) {
+			RTE_LOG(DEBUG, POWER, "Failed to read %s\n",
+					POWER_UNCORE_SYSFILE_BASE_MIN_FREQ);
+			goto err;
+		}
+	}
+
+	/* Curr min */
+	open_core_sysfs_file(&f_min, "rw+", POWER_UNCORE_SYSFILE_MIN_FREQ,
+			ui->pkg, ui->die);
+	if (f_min == NULL) {
+		RTE_LOG(DEBUG, POWER, "failed to open %s\n",
+				POWER_UNCORE_SYSFILE_MIN_FREQ);
+		goto err;
+	}
+	if (f_min != NULL) {
+		ret = read_core_sysfs_u32(f_min, &min_freq);
+		if (ret < 0) {
+			RTE_LOG(DEBUG, POWER, "Failed to read %s\n",
+					POWER_UNCORE_SYSFILE_MIN_FREQ);
+			goto err;
+		}
+	}
+
+	/* Curr max */
+	open_core_sysfs_file(&f_max, "rw+", POWER_UNCORE_SYSFILE_MAX_FREQ,
+			ui->pkg, ui->die);
+	if (f_max == NULL) {
+		RTE_LOG(DEBUG, POWER, "failed to open %s\n",
+				POWER_UNCORE_SYSFILE_MAX_FREQ);
+		goto err;
+	}
+	if (f_max != NULL) {
+		ret = read_core_sysfs_u32(f_max, &max_freq);
+		if (ret < 0) {
+			RTE_LOG(DEBUG, POWER, "Failed to read %s\n",
+					POWER_UNCORE_SYSFILE_MAX_FREQ);
+			goto err;
+		}
+	}
+
+	/* assign file handles */
+	ui->f_cur_min = f_min;
+	ui->f_cur_max = f_max;
+	ui->f_base_max = f_base_max;
+	ui->f_base_min = f_base_min;
+	ui->init_max_freq = base_max_freq;
+	ui->init_min_freq = base_min_freq;
+
+	return 0;
+
+err:
+	if (f_base_min != NULL)
+		fclose(f_base_min);
+	if (f_base_max != NULL)
+		fclose(f_base_max);
+	if (f_min != NULL)
+		fclose(f_min);
+	if (f_max != NULL)
+		fclose(f_max);
+	return -1;
+}
+
+/**
+ * Get the available uncore frequencies of the specific die by reading the
+ * sys file.
+ */
+static int
+power_get_available_uncore_freqs(struct uncore_power_info *ui)
+{
+	int ret = -1;
+	uint32_t i, num_uncore_freqs = 0;
+
+	num_uncore_freqs = (ui->init_max_freq - ui->init_min_freq) / BUS_FREQ + 1;
+	if (num_uncore_freqs >= RTE_MAX_UNCORE_FREQS) {
+		RTE_LOG(ERR, POWER, "Too many available uncore frequencies: %d\n",
+				num_uncore_freqs);
+		goto out;
+	}
+
+	/* Generate the uncore freq bucket array. */
+	for (i = 0; i < num_uncore_freqs; i++)
+		ui->freqs[i] = ui->init_max_freq - (i) * BUS_FREQ;
+
+	ui->nb_freqs = num_uncore_freqs;
+
+	ret = 0;
+
+	POWER_DEBUG_TRACE("%d uncore frequency(s) of pkg %02u die %02u are available\n",
+			num_uncore_freqs, ui->pkg, ui->die);
+
+out:
+	return ret;
+}
+
+
+int
+rte_power_uncore_init(unsigned int pkg, unsigned int die)
+{
+	struct uncore_power_info *ui;
+
+	/* Check if pkg and die values are viable */
+	if (pkg >= RTE_MAX_NUMA_NODES) {
+		RTE_LOG(DEBUG, POWER, "Package number %02u can not exceed %u\n",
+				pkg, RTE_MAX_NUMA_NODES - 1U);
+		return -1;
+	}
+
+	if (die >= RTE_MAX_NUMA_DIE) {
+		RTE_LOG(DEBUG, POWER, "Die number %02u can not exceed %u\n",
+				die, RTE_MAX_NUMA_DIE - 1U);
+		return -1;
+	}
+
+	ui = &uncore_info[pkg][die];
+	ui->die = die;
+	ui->pkg = pkg;
+
+	/* Init for setting unocre die frequency */
+	if (power_init_for_setting_uncore_freq(ui) < 0) {
+		RTE_LOG(DEBUG, POWER, "Cannot init for setting uncore frequency for "
+				"pkg %02u die %02u\n", pkg, die);
+		return -1;
+	}
+
+	/* Get the available frequencies */
+	if (power_get_available_uncore_freqs(ui) < 0) {
+		RTE_LOG(DEBUG, POWER, "Cannot get available uncore frequencies of "
+				"pkg %02u die %02u\n", pkg, die);
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+rte_power_uncore_exit(unsigned int pkg, unsigned int die)
+{
+	struct uncore_power_info *ui;
+
+	if (pkg >= RTE_MAX_NUMA_NODES) {
+		RTE_LOG(DEBUG, POWER, "Package number %02u can not exceed %u\n",
+				pkg, RTE_MAX_NUMA_NODES - 1U);
+		return -1;
+	}
+
+	if (die >= RTE_MAX_NUMA_DIE) {
+		RTE_LOG(DEBUG, POWER, "Die number %02u can not exceed %u\n",
+				die, RTE_MAX_NUMA_DIE - 1U);
+		return -1;
+	}
+
+	ui = &uncore_info[pkg][die];
+
+	/* Close FD of setting freq */
+	fclose(ui->f_cur_min);
+	fclose(ui->f_cur_max);
+	fclose(ui->f_base_max);
+	fclose(ui->f_base_min);
+	ui->f_cur_min = NULL;
+	ui->f_cur_max = NULL;
+	ui->f_base_min = NULL;
+	ui->f_base_max = NULL;
+
+	return 0;
+}
+
+int
+rte_power_get_uncore_freq(unsigned int pkg, unsigned int die)
+{
+	if (pkg >= RTE_MAX_NUMA_NODES) {
+		RTE_LOG(DEBUG, POWER, "Invalid package number\n");
+		return -1;
+	}
+
+	if (die >= RTE_MAX_NUMA_DIE) {
+		RTE_LOG(DEBUG, POWER, "Invalid die number\n");
+		return -1;
+	}
+
+	return uncore_info[pkg][die].cur_idx;
+}
+
+int
+rte_power_set_uncore_freq(unsigned int pkg, unsigned int die, uint32_t index)
+{
+	if (pkg >= RTE_MAX_NUMA_NODES) {
+		RTE_LOG(DEBUG, POWER, "Invalid package number\n");
+		return -1;
+	}
+
+	if (die >= RTE_MAX_NUMA_DIE) {
+		RTE_LOG(DEBUG, POWER, "Invalid die number\n");
+		return -1;
+	}
+
+	return set_uncore_freq_internal(&(uncore_info[pkg][die]), index);
+}
+
+int
+rte_power_uncore_freq_max(unsigned int pkg, unsigned int die)
+{
+	if (pkg >= RTE_MAX_NUMA_NODES) {
+		RTE_LOG(DEBUG, POWER, "Invalid package number\n");
+		return -1;
+	}
+
+	if (die >= RTE_MAX_NUMA_DIE) {
+		RTE_LOG(DEBUG, POWER, "Invalid die number\n");
+		return -1;
+	}
+
+	struct uncore_power_info *ui = &uncore_info[pkg][die];
+
+	if (fprintf(ui->f_cur_max, "%u", 0) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new uncore frequency for "
+				"pkg %02u die %02u\n", ui->pkg, ui->die);
+		return -1;
+	}
+
+	fflush(ui->f_cur_max);
+	return 0;
+}
+
+
+int
+rte_power_uncore_freq_min(unsigned int pkg, unsigned int die)
+{
+	if (pkg >= RTE_MAX_NUMA_NODES) {
+		RTE_LOG(DEBUG, POWER, "Invalid package number\n");
+		return -1;
+	}
+
+	if (die >= RTE_MAX_NUMA_DIE) {
+		RTE_LOG(DEBUG, POWER, "Invalid die number\n");
+		return -1;
+	}
+
+	struct uncore_power_info *ui = &uncore_info[pkg][die];
+
+	if (fprintf(ui->f_cur_min, "%u",  ui->freqs[ui->nb_freqs - 1]) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new uncore frequency for "
+				"pkg %02u die %02u\n", ui->pkg, ui->die);
+		return -1;
+	}
+
+	fflush(ui->f_cur_min);
+	return 0;
+}
+
+int
+rte_power_uncore_get_num_freqs(unsigned int pkg, unsigned int die)
+{
+	if (pkg >= RTE_MAX_NUMA_NODES) {
+		RTE_LOG(DEBUG, POWER, "Invalid package number\n");
+		return -1;
+	}
+
+	if (die >= RTE_MAX_NUMA_DIE) {
+		RTE_LOG(DEBUG, POWER, "Invalid die number\n");
+		return -1;
+	}
+
+	return uncore_info[pkg][die].nb_freqs;
+}
diff --git a/lib/power/rte_power_uncore.h b/lib/power/rte_power_uncore.h
new file mode 100644
index 0000000000..6376c3da11
--- /dev/null
+++ b/lib/power/rte_power_uncore.h
@@ -0,0 +1,159 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2022 Intel Corporation
+ */
+
+#ifndef _RTE_POWER_UNCORE_H
+#define _RTE_POWER_UNCORE_H
+
+/**
+ * @file
+ * RTE Uncore Frequency Management
+ */
+
+#include "rte_power.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Initialize uncore frequency management for specific die on a package. It will check and set the
+ * governor to performance for the die, get the available frequencies, and
+ * prepare to set new die frequency.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+__rte_experimental
+int
+rte_power_uncore_init(unsigned int pkg, unsigned int die);
+
+/**
+ * Exit uncore frequency management on a specific die on a package. It will set the governor to
+ * which is before initialized.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+__rte_experimental
+int
+rte_power_uncore_exit(unsigned int pkg, unsigned int die);
+
+/**
+ * Return the current index of available frequencies of a specific die on a package.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ *
+ * @return
+ *  - The current index of available frequencies.
+ *  - Negative on error.
+ */
+__rte_experimental
+int
+rte_power_get_uncore_freq(unsigned int pkg, unsigned int die);
+
+/**
+ * Set the new frequency for a specific die on a package by indicating the index of
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ * @param index
+ *  The index of available frequencies.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+__rte_experimental
+int
+rte_power_set_uncore_freq(unsigned int pkg, unsigned int die, uint32_t index);
+
+/**
+ * Scale up the frequency of a specific die on a package to the highest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+__rte_experimental
+int
+rte_power_uncore_freq_max(unsigned int pkg, unsigned int die);
+
+/**
+ * Scale down the frequency of a specific die on a package to the lowest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+__rte_experimental
+int
+rte_power_uncore_freq_min(unsigned int pkg, unsigned int die);
+
+/**
+ * Return the list length of available frequencies in the index array
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ *
+ * @return
+ *  - The number of available index's in frequency array
+ *  - Negative on error
+ */
+__rte_experimental
+int
+rte_power_uncore_get_num_freqs(unsigned int pkg, unsigned int die);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/power/version.map b/lib/power/version.map
index a687754f4a..e9681df291 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -48,4 +48,11 @@ EXPERIMENTAL {
 	rte_power_pmd_mgmt_set_pause_duration;
 	rte_power_pmd_mgmt_set_scaling_freq_max;
 	rte_power_pmd_mgmt_set_scaling_freq_min;
+	rte_power_uncore_init;
+	rte_power_uncore_exit;
+	rte_power_get_uncore_freq;
+	rte_power_set_uncore_freq;
+	rte_power_uncore_freq_max;
+	rte_power_uncore_freq_min;
+	rte_power_uncore_get_num_freqs;
 };
-- 
2.25.1

RE: [PATCH v2 1/3] power: add uncore API to power library

[Pattan, Reshma]

> -----Original Message-----
> From: Kearney, Tadhg <tadhg.kearney@intel.com>
> diff --git a/config/x86/meson.build b/config/x86/meson.build index
> +dpdk_conf.set('RTE_MAX_NUMA_DIE', 1)
> +dpdk_conf.set('RTE_MAX_UNCORE_FREQS', 32)

Check if these flags should be added to other platforms.

<snip>


> +Abstract
> +~~~~~~~~
> +Up to 60% power saving can be achieved by reducing the uncore frequency
> to its lowest value.

Might be give some intro on this before specifying directly the power saving.  Would be good to say power savings achieved instead of specific with any %.
Also

> +With later kernels, there is now a sysfs entry to allow adjustment of uncore
> frequency.

Kernel provides the driver "intel-uncore-frequency to control the uncore frequency limits for x86 platform. The driver is available from the kernel version 5.6 and above.

> diff --git a/lib/power/rte_power_uncore.c b/lib/power/rte_power_uncore.c
> +static int
> +set_uncore_freq_internal(struct uncore_power_info *ui, uint32_t idx) {
<snip>

> +	open_core_sysfs_file(&ui->f_cur_max, "rw+",
> POWER_UNCORE_SYSFILE_MAX_FREQ,

You have already this f_cur_max opened , you can re-use, instead of opening again.

Thanks,
Reshma

Re: [PATCH v2 1/3] power: add uncore API to power library

[Bruce Richardson]

On Wed, Jul 20, 2022 at 04:10:07PM +0000, Pattan, Reshma wrote:
> 
> 
> > -----Original Message-----
> > From: Kearney, Tadhg <tadhg.kearney@intel.com>
> > diff --git a/config/x86/meson.build b/config/x86/meson.build index
> > +dpdk_conf.set('RTE_MAX_NUMA_DIE', 1)
> > +dpdk_conf.set('RTE_MAX_UNCORE_FREQS', 32)
> 
> Check if these flags should be added to other platforms.
> 

I think we need some explanation as to what these values are for, and how
they should be set for various platforms. Are they always these values, or
should they be determined programmatically at build time?

/Bruce

RE: [PATCH v2 1/3] power: add uncore API to power library

[Kearney, Tadhg]

> -----Original Message-----
> From: Richardson, Bruce <bruce.richardson@intel.com>
*snip*
> > > +dpdk_conf.set('RTE_MAX_NUMA_DIE', 1)
> > > +dpdk_conf.set('RTE_MAX_UNCORE_FREQS', 32)
> >
> > Check if these flags should be added to other platforms.
> >
> 
> I think we need some explanation as to what these values are for, and how
> they should be set for various platforms.

Kernel does support sysfs uncore interface, as in below: 
"To control uncore frequency, a sysfs interface is provided in the directory: /sys/devices/system/cpu/intel_uncore_frequency/.
There is one directory for each package and die combination as the scope of uncore scaling control is per die in multiple die/package SoCs or per package for single die per package SoCs. The name represents the scope of control. For example: 'package_00_die_00' is for package id 0 and die 0." 

The macros 'RTE_MAX_NUMA_DIE' and 'RTE_MAX_UNCORE_FREQS' were first introduced as we needed to impose limits on cpu die number and how many uncore frequencies would be allowed for a system.

> Are they always these values, or

No, they differ from system to system.

> should they be determined programmatically at build time?

Retrieving information about die and number of uncore frequencies is not possible as we do not have a direct interface in Linux which would give this information.

With all this, better to remove these macros altogether from config path and instead figure out number of dies during run time, by iterating through package and die files from the sysfs file  'package_*_die_*',
  for systems that support sysfs uncore path. We would then give error with systems that do not contain sysfs uncore path.

Regards, Tadhg
 

[PATCH v2 2/3] test/power: add unit tests for uncore API

[Tadhg Kearney]

Add basic unit tests covering all seven uncore api's.

Signed-off-by: Tadhg Kearney <tadhg.kearney@intel.com>
---
 app/test/meson.build         |   2 +
 app/test/test_power_uncore.c | 245 +++++++++++++++++++++++++++++++++++
 2 files changed, 247 insertions(+)
 create mode 100644 app/test/test_power_uncore.c

diff --git a/app/test/meson.build b/app/test/meson.build
index 431c5bd318..3ab360ecee 100644
--- a/app/test/meson.build
+++ b/app/test/meson.build
@@ -100,6 +100,7 @@ test_sources = files(
         'test_power.c',
         'test_power_cpufreq.c',
         'test_power_kvm_vm.c',
+        'test_power_uncore.c',
         'test_prefetch.c',
         'test_rand_perf.c',
         'test_rawdev.c',
@@ -240,6 +241,7 @@ fast_tests = [
         ['power_cpufreq_autotest', false, true],
         ['power_autotest', true, true],
         ['power_kvm_vm_autotest', false, true],
+        ['power_uncore_autotest', true, true],
         ['reorder_autotest', true, true],
         ['service_autotest', true, true],
         ['thash_autotest', true, true],
diff --git a/app/test/test_power_uncore.c b/app/test/test_power_uncore.c
new file mode 100644
index 0000000000..6dbfe22cb6
--- /dev/null
+++ b/app/test/test_power_uncore.c
@@ -0,0 +1,245 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2022 Intel Corporation
+ */
+
+#include "test.h"
+
+#ifndef RTE_LIB_POWER
+
+static int
+test_power_uncore(void)
+{
+	printf("Power management library not supported, skipping test\n");
+	return TEST_SKIPPED;
+}
+
+#else
+#include <rte_power_uncore.h>
+#include <power_common.h>
+
+#define VALID_PKG 0
+#define VALID_DIE 0
+#define INVALID_PKG (RTE_MAX_NUMA_NODES + 1)
+#define INVALID_DIE (RTE_MAX_NUMA_DIE + 1)
+#define VALID_INDEX 1
+#define INVALID_INDEX (RTE_MAX_UNCORE_FREQS + 1)
+
+static int check_power_uncore_init(void)
+{
+	int ret;
+
+	/* Test initialisation of uncore configuration*/
+	ret = rte_power_uncore_init(VALID_PKG, VALID_DIE);
+	if (ret < 0) {
+		printf("Cannot initialise uncore power management for pkg %u die %u, this "
+			"may occur if environment is not configured "
+			"correctly(APCI cpufreq) or operating in another valid "
+			"Power management environment\n", VALID_PKG, VALID_DIE);
+		return -1;
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_uncore_init(INVALID_PKG, INVALID_DIE);
+	if (ret == 0) {
+		printf("Unexpectedly was able to initialise uncore power management "
+			"for pkg %u die %u\n", INVALID_PKG, INVALID_DIE);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_get_uncore_freq(void)
+{
+	int ret;
+
+	/* Successfully get uncore freq */
+	ret = rte_power_get_uncore_freq(VALID_PKG, VALID_DIE);
+	if (ret < 0) {
+		printf("Failed to get uncore frequency for pkg %u die %u\n",
+							VALID_PKG, VALID_DIE);
+		return -1;
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_get_uncore_freq(INVALID_PKG, INVALID_DIE);
+	if (ret >= 0) {
+		printf("Unexpectedly got invalid uncore frequency for pkg %u die %u\n",
+							INVALID_PKG, INVALID_DIE);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_set_uncore_freq(void)
+{
+	int ret;
+
+	/* Successfully set uncore freq */
+	ret = rte_power_set_uncore_freq(VALID_PKG, VALID_DIE, VALID_INDEX);
+	if (ret < 0) {
+		printf("Failed to set uncore frequency for pkg %u die %u index %u\n",
+							VALID_PKG, VALID_DIE, VALID_INDEX);
+		return -1;
+	}
+
+	/* Try to unsuccessfully set invalid uncore freq index */
+	ret = rte_power_set_uncore_freq(VALID_PKG, VALID_DIE, INVALID_INDEX);
+	if (ret == 0) {
+		printf("Unexpectedly set invalid uncore index for pkg %u die %u index %u\n",
+							VALID_PKG, VALID_DIE, INVALID_INDEX);
+		return -1;
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_set_uncore_freq(INVALID_PKG, INVALID_DIE, VALID_INDEX);
+	if (ret == 0) {
+		printf("Unexpectedly set invalid uncore frequency for pkg %u die %u index %u\n",
+							INVALID_PKG, INVALID_DIE, VALID_INDEX);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_uncore_freq_max(void)
+{
+	int ret;
+
+	/* Successfully get max uncore freq */
+	ret = rte_power_uncore_freq_max(VALID_PKG, VALID_DIE);
+	if (ret < 0) {
+		printf("Failed to set max uncore frequency for pkg %u die %u\n",
+							VALID_PKG, VALID_DIE);
+		return -1;
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_uncore_freq_max(INVALID_PKG, INVALID_DIE);
+	if (ret == 0) {
+		printf("Unexpectedly set invalid max uncore frequency for pkg %u die %u\n",
+							INVALID_PKG, INVALID_DIE);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_uncore_freq_min(void)
+{
+	int ret;
+
+	/* Successfully get min uncore freq */
+	ret = rte_power_uncore_freq_min(VALID_PKG, VALID_DIE);
+	if (ret < 0) {
+		printf("Failed to set min uncore frequency for pkg %u die %u\n",
+							VALID_PKG, VALID_DIE);
+		return -1;
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_uncore_freq_min(INVALID_PKG, INVALID_DIE);
+	if (ret == 0) {
+		printf("Unexpectedly set invalid min uncore frequency for pkg %u die %u\n",
+							INVALID_PKG, INVALID_DIE);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_uncore_get_num_freqs(void)
+{
+	int ret;
+
+	/* Successfully get number of uncore freq */
+	ret = rte_power_uncore_get_num_freqs(VALID_PKG, VALID_DIE);
+	if (ret < 0) {
+		printf("Failed to get number of uncore frequencies for pkg %u die %u\n",
+							VALID_PKG, VALID_DIE);
+		return -1;
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_uncore_get_num_freqs(INVALID_PKG, INVALID_DIE);
+	if (ret >= 0) {
+		printf("Unexpectedly got number of invalid frequencies for pkg %u die %u\n",
+							INVALID_PKG, INVALID_DIE);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_uncore_exit(void)
+{
+	int ret;
+
+	/* Successfully exit uncore power management */
+	ret = rte_power_uncore_exit(VALID_PKG, VALID_DIE);
+	if (ret < 0) {
+		printf("Failed to exit uncore power management for pkg %u die %u\n",
+							VALID_PKG, VALID_DIE);
+		return -1;
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_uncore_exit(INVALID_PKG, INVALID_DIE);
+	if (ret == 0) {
+		printf("Unexpectedly was able to exit uncore power management for pkg %u die %u\n",
+							INVALID_PKG, INVALID_DIE);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+test_power_uncore(void)
+{
+	int ret;
+
+	ret = check_power_uncore_init();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_get_uncore_freq();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_set_uncore_freq();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_uncore_freq_max();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_uncore_freq_min();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_uncore_get_num_freqs();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_uncore_exit();
+	if (ret < 0)
+		return -1;
+
+	return 0;
+
+fail_all:
+	rte_power_uncore_exit(VALID_PKG, VALID_DIE);
+	return -1;
+}
+#endif
+
+REGISTER_TEST_COMMAND(power_uncore_autotest, test_power_uncore);
-- 
2.25.1

[PATCH v2 3/3] l3fwd-power: add option to call uncore API

[Tadhg Kearney]

Add option for setting uncore frequency min/max/index, through uncore api.
This will be set for each package and die on the SKU. On exit, uncore
frequency will be reverted back to previous frequency.

Signed-off-by: Tadhg Kearney <tadhg.kearney@intel.com>
---
 .../sample_app_ug/l3_forward_power_man.rst    |  28 +++
 examples/l3fwd-power/main.c                   | 190 ++++++++++++++++--
 2 files changed, 202 insertions(+), 16 deletions(-)

diff --git a/doc/guides/sample_app_ug/l3_forward_power_man.rst b/doc/guides/sample_app_ug/l3_forward_power_man.rst
index 8f6d906200..1e452140a1 100644
--- a/doc/guides/sample_app_ug/l3_forward_power_man.rst
+++ b/doc/guides/sample_app_ug/l3_forward_power_man.rst
@@ -97,6 +97,12 @@ where,
 *   -P: Sets all ports to promiscuous mode so that packets are accepted regardless of the packet's Ethernet MAC destination address.
     Without this option, only packets with the Ethernet MAC destination address set to the Ethernet address of the port are accepted.
 
+*   -u: optional, sets uncore frequency to minimum value.
+
+*   -U: optional, sets uncore frequency to maximum value.
+
+*   -i (frequency index): optional, sets uncore frequency to frequency index value, by setting min and max values to be the same.
+
 *   --config (port,queue,lcore)[,(port,queue,lcore)]: determines which queues from which ports are mapped to which cores.
 
 *   --max-pkt-len: optional, maximum packet length in decimal (64-9600)
@@ -364,3 +370,25 @@ in the DPDK Programmer's Guide for more details on PMD power management.
 .. code-block:: console
 
         ./<build_dir>/examples/dpdk-l3fwd-power -l 1-3 -- -p 0x0f --config="(0,0,2),(0,1,3)" --pmd-mgmt=scale
+
+Setting Uncore Values
+---------------------
+
+Uncore frequency can be adjusted through manipulating related sysfs entries to adjust the minimum and maximum uncore values.
+This will be set for each package and die on the SKU. Three options are available for setting uncore frequency:
+
+``-u``
+  This will set uncore to minimum frequency possible.
+
+``-U``
+  This will set uncore to maximum frequency possible.
+
+``-i``
+  This will allow you to set the specific uncore frequency index that you want, by setting
+  minimum and maximum values to be the same. Frequency index's are set 100000Hz apart from
+  maximum to minimum.
+  Frequency index values are in descending order, ie, index 0 is maximum frequency index.
+
+.. code-block:: console
+
+        ./<build_dir>/examples/dpdk-l3fwd-power -l 1-3 -- -p 0x0f --config="(0,0,2),(0,1,3)" -i 1
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 887c6eae3f..5f74e29e3a 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -15,6 +15,8 @@
 #include <unistd.h>
 #include <signal.h>
 #include <math.h>
+#include <dirent.h>
+#include <fnmatch.h>
 
 #include <rte_common.h>
 #include <rte_byteorder.h>
@@ -47,6 +49,7 @@
 #include <rte_metrics.h>
 #include <rte_telemetry.h>
 #include <rte_power_pmd_mgmt.h>
+#include <rte_power_uncore.h>
 
 #include "perf_core.h"
 #include "main.h"
@@ -71,6 +74,7 @@
 
 #ifndef APP_LOOKUP_METHOD
 #define APP_LOOKUP_METHOD             APP_LOOKUP_LPM
+
 #endif
 
 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
@@ -83,7 +87,7 @@
 
 #ifndef IPv6_BYTES
 #define IPv6_BYTES_FMT "%02x%02x:%02x%02x:%02x%02x:%02x%02x:"\
-                       "%02x%02x:%02x%02x:%02x%02x:%02x%02x"
+					   "%02x%02x:%02x%02x:%02x%02x:%02x%02x"
 #define IPv6_BYTES(addr) \
 	addr[0],  addr[1], addr[2],  addr[3], \
 	addr[4],  addr[5], addr[6],  addr[7], \
@@ -134,9 +138,14 @@
 
 #define NUM_TELSTATS RTE_DIM(telstats_strings)
 
+#define UNCORE_FREQUENCY_DIR "/sys/devices/system/cpu/intel_uncore_frequency"
+
 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
 
+/* Max number of nodes times dies available on uncore */
+#define MAX_DIE_NODES (RTE_MAX_NUMA_DIE * RTE_MAX_NUMA_NODES)
+
 /* ethernet addresses of ports */
 static struct rte_ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
 
@@ -145,6 +154,8 @@ static rte_spinlock_t locks[RTE_MAX_ETHPORTS];
 
 /* mask of enabled ports */
 static uint32_t enabled_port_mask = 0;
+/* if uncore frequency was enabled without errors */
+static int enabled_uncore;
 /* Ports set in promiscuous mode off by default. */
 static int promiscuous_on = 0;
 /* NUMA is enabled by default. */
@@ -165,6 +176,13 @@ struct telstats_name {
 	char name[RTE_ETH_XSTATS_NAME_SIZE];
 };
 
+struct uncore_info {
+	unsigned int pkg;
+	unsigned int die;
+};
+
+struct uncore_info ui[MAX_DIE_NODES];
+
 /* telemetry stats to be reported */
 const struct telstats_name telstats_strings[] = {
 	{"empty_poll"},
@@ -557,9 +575,9 @@ static void
 print_ipv6_key(struct ipv6_5tuple key)
 {
 	printf( "IP dst = " IPv6_BYTES_FMT ", IP src = " IPv6_BYTES_FMT ", "
-	        "port dst = %d, port src = %d, proto = %d\n",
-	        IPv6_BYTES(key.ip_dst), IPv6_BYTES(key.ip_src),
-	        key.port_dst, key.port_src, key.proto);
+			"port dst = %d, port src = %d, proto = %d\n",
+			IPv6_BYTES(key.ip_dst), IPv6_BYTES(key.ip_src),
+			key.port_dst, key.port_src, key.proto);
 }
 
 static inline uint16_t
@@ -674,9 +692,9 @@ parse_ptype_one(struct rte_mbuf *m)
 
 static uint16_t
 cb_parse_ptype(uint16_t port __rte_unused, uint16_t queue __rte_unused,
-	       struct rte_mbuf *pkts[], uint16_t nb_pkts,
-	       uint16_t max_pkts __rte_unused,
-	       void *user_param __rte_unused)
+		   struct rte_mbuf *pkts[], uint16_t nb_pkts,
+		   uint16_t max_pkts __rte_unused,
+		   void *user_param __rte_unused)
 {
 	unsigned int i;
 
@@ -797,8 +815,8 @@ power_idle_heuristic(uint32_t zero_rx_packet_count)
 
 static inline enum freq_scale_hint_t
 power_freq_scaleup_heuristic(unsigned lcore_id,
-			     uint16_t port_id,
-			     uint16_t queue_id)
+				 uint16_t port_id,
+				 uint16_t queue_id)
 {
 	uint32_t rxq_count = rte_eth_rx_queue_count(port_id, queue_id);
 /**
@@ -1051,7 +1069,7 @@ static int main_intr_loop(__rte_unused void *dummy)
 			 * less as possible.
 			 */
 			for (i = 1,
-			    lcore_idle_hint = qconf->rx_queue_list[0].idle_hint;
+				lcore_idle_hint = qconf->rx_queue_list[0].idle_hint;
 					i < qconf->n_rx_queue; ++i) {
 				rx_queue = &(qconf->rx_queue_list[i]);
 				if (rx_queue->idle_hint < lcore_idle_hint)
@@ -1616,6 +1634,9 @@ print_usage(const char *prgname)
 		"  [--max-pkt-len PKTLEN]\n"
 		"  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
 		"  -P: enable promiscuous mode\n"
+		"  -u: set min frequency for uncore\n"
+		"  -U: set max frequency for uncore\n"
+		"  -i (frequency index): set frequency index for uncore\n"
 		"  --config (port,queue,lcore): rx queues configuration\n"
 		"  --high-perf-cores CORELIST: list of high performance cores\n"
 		"  --perf-config: similar as config, cores specified as indices"
@@ -1672,6 +1693,112 @@ static int parse_max_pkt_len(const char *pktlen)
 	return len;
 }
 
+static int
+parse_uncore_dir(void)
+{
+	DIR *d;
+	struct dirent *dir;
+	char pkg[3], die[3];
+	unsigned int count = 0;
+	static const char filter[] = "*freq_khz*";
+
+	d = opendir(UNCORE_FREQUENCY_DIR);
+	if (!d) {
+		RTE_LOG(ERR, EAL, "Unable to open uncore frequency directory");
+		return -1;
+	}
+
+	else {
+		/* Loop through every file name in uncore frequency dir extracting
+		 * pkg + die numbers from file names.
+		 */
+		while ((dir = readdir(d)) != NULL) {
+			if (fnmatch(filter, dir->d_name, 0) > 0) {
+				/* Extract pkg + die numbers from string 'package_XX_die_XX' */
+				sprintf(pkg, "%c%c", dir->d_name[8], dir->d_name[9]);
+				sprintf(die, "%c%c", dir->d_name[15], dir->d_name[16]);
+				ui[count].pkg = atoi(pkg);
+				ui[count].die = atoi(die);
+				count++;
+			}
+		}
+		closedir(d);
+	}
+	return(count);
+}
+
+static int
+parse_uncore_min_max(const char *choice)
+{
+	int ret = 0, total_files, i;
+	unsigned int die, pkg;
+	total_files = parse_uncore_dir();
+
+	if (total_files < 1)
+		return -1;
+
+	for (i = 0; i < total_files; i++) {
+		die = ui[i].die;
+		pkg = ui[i].pkg;
+		rte_power_uncore_init(pkg, die);
+		if (!strncmp(choice, "max", 3)) {
+			ret = rte_power_uncore_freq_max(pkg, die);
+			if (ret == -1)
+				RTE_LOG(INFO, L3FWD_POWER, "Unable to set max uncore value for pkg %02u die %02u\n"\
+				, pkg, die);
+		}
+
+		else if (!strncmp(choice, "min", 3)) {
+			ret = rte_power_uncore_freq_min(pkg, die);
+			if (ret == -1) {
+				RTE_LOG(INFO, L3FWD_POWER, "Unable to set min uncore value for pkg %02u die %02u\n"\
+				, pkg, die);
+				return ret;
+			}
+		} else {
+			RTE_LOG(INFO, L3FWD_POWER, "Uncore choice provided invalid\n");
+			return -1;
+		}
+	}
+
+	RTE_LOG(INFO, L3FWD_POWER, "Successfully set max/min uncore frequency.\n");
+	return ret;
+}
+
+static int
+parse_uncore_index(const char *choice)
+{
+	int ret = 0, total_files, i;
+	unsigned int  die, pkg;
+	total_files = parse_uncore_dir();
+
+	if (total_files < 1)
+		return -1;
+
+	for (i = 0; i < total_files; i++) {
+		die = ui[i].die;
+		pkg = ui[i].pkg;
+		rte_power_uncore_init(pkg, die);
+
+		int frequency_index = atoi(choice);
+		int freq_array_len = rte_power_uncore_get_num_freqs(pkg, die);
+		if (frequency_index > freq_array_len) {
+			RTE_LOG(INFO, L3FWD_POWER, \
+			"Frequency index given out of range, please choose a value from 0 to %d.\n" \
+			, freq_array_len);
+			return -1;
+		}
+		ret = rte_power_set_uncore_freq(pkg, die, frequency_index);
+		if (ret == -1) {
+			RTE_LOG(INFO, L3FWD_POWER, \
+			"Unable to set uncore index value for pkg %02u die %02u\n", pkg, die);
+			return ret;
+		}
+	}
+	RTE_LOG(INFO, L3FWD_POWER, "Successfully set freq index for uncore\n");
+	return ret;
+}
+
 static int
 parse_portmask(const char *portmask)
 {
@@ -1861,10 +1988,12 @@ parse_args(int argc, char **argv)
 		{CMD_LINE_OPT_SCALE_FREQ_MAX, 1, 0, 0},
 		{NULL, 0, 0, 0}
 	};
+	const char *min = "min";
+	const char *max = "max";
 
 	argvopt = argv;
 
-	while ((opt = getopt_long(argc, argvopt, "p:l:m:h:P",
+	while ((opt = getopt_long(argc, argvopt, "p:l:m:h:P:u U i:",
 				lgopts, &option_index)) != EOF) {
 
 		switch (opt) {
@@ -1893,6 +2022,30 @@ parse_args(int argc, char **argv)
 			limit = parse_max_pkt_len(optarg);
 			freq_tlb[HGH] = limit;
 			break;
+		case 'u':
+			enabled_uncore = parse_uncore_min_max(min);
+			if (enabled_uncore < 0) {
+				printf("Unable to reach min uncore value\n");
+				print_usage(prgname);
+				return -1;
+			}
+			break;
+		case 'U':
+			enabled_uncore = parse_uncore_min_max(max);
+			if (enabled_uncore < 0) {
+				printf("Unable to reach max uncore value\n");
+				print_usage(prgname);
+				return -1;
+			}
+			break;
+		case 'i':
+			enabled_uncore = parse_uncore_index(optarg);
+			if (enabled_uncore < 0) {
+				printf("Unable to reach uncore index value\n");
+				print_usage(prgname);
+				return -1;
+			}
+			break;
 		/* long options */
 		case 0:
 			if (!strncmp(lgopts[option_index].name, "config", 6)) {
@@ -2003,8 +2156,8 @@ parse_args(int argc, char **argv)
 			}
 
 			if (!strncmp(lgopts[option_index].name,
-				     CMD_LINE_OPT_PARSE_PTYPE,
-				     sizeof(CMD_LINE_OPT_PARSE_PTYPE))) {
+					 CMD_LINE_OPT_PARSE_PTYPE,
+					 sizeof(CMD_LINE_OPT_PARSE_PTYPE))) {
 				printf("soft parse-ptype is enabled\n");
 				parse_ptype = 1;
 			}
@@ -2262,7 +2415,7 @@ check_all_ports_link_status(uint32_t port_mask)
 				rte_eth_link_to_str(link_status_text,
 					sizeof(link_status_text), &link);
 				printf("Port %d %s\n", portid,
-				       link_status_text);
+					   link_status_text);
 				continue;
 			}
 			/* clear all_ports_up flag if any link down */
@@ -2365,6 +2518,8 @@ static int
 deinit_power_library(void)
 {
 	unsigned int lcore_id;
+	int i, total_files;
+	total_files = parse_uncore_dir();
 	int ret = 0;
 
 	RTE_LCORE_FOREACH(lcore_id) {
@@ -2377,6 +2532,9 @@ deinit_power_library(void)
 			return ret;
 		}
 	}
+
+	for (i = 0; i < total_files; i++)
+		rte_power_uncore_exit(ui[i].pkg, ui[i].die);
 	return ret;
 }
 
@@ -2734,7 +2892,7 @@ main(int argc, char **argv)
 					"err=%d, port=%d\n", ret, portid);
 
 		ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd,
-						       &nb_txd);
+							   &nb_txd);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE,
 				 "Cannot adjust number of descriptors: err=%d, port=%d\n",
@@ -2791,7 +2949,7 @@ main(int argc, char **argv)
 			txconf = &dev_info.default_txconf;
 			txconf->offloads = local_port_conf.txmode.offloads;
 			ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
-						     socketid, txconf);
+							 socketid, txconf);
 			if (ret < 0)
 				rte_exit(EXIT_FAILURE,
 					"rte_eth_tx_queue_setup: err=%d, "
-- 
2.25.1

RE: [PATCH v2 3/3] l3fwd-power: add option to call uncore API

[Pattan, Reshma]

> -----Original Message-----
> From: Kearney, Tadhg <tadhg.kearney@intel.com>

<snip>

> +*   -i (frequency index): optional, sets uncore frequency to frequency index
> value, by setting min and max values to be the same.
> +

This is not optional argument. Need to remove optional in the help text of the parameter.


> +``-i``
> +  This will allow you to set the specific uncore frequency index that
> +you want, by setting
> +  minimum and maximum values to be the same. Frequency index's are set
> +100000Hz apart from
> +  maximum to minimum.
> +  Frequency index values are in descending order, ie, index 0 is maximum
> frequency index.
> +

I would say by setting the uncore frequency  to a frequency pointed by index . The frequencies at each index differ by 100MHz. The value you have mentioned in 100Khz.


<snip>
>  #define IPv6_BYTES_FMT "%02x%02x:%02x%02x:%02x%02x:%02x%02x:"\
> -                       "%02x%02x:%02x%02x:%02x%02x:%02x%02x"
> +

This change looks to be accidental. Undo this change.

>  	printf( "IP dst = " IPv6_BYTES_FMT ", IP src = " IPv6_BYTES_FMT ", "
> -	        "port dst = %d, port src = %d, proto = %d\n",
> -	        IPv6_BYTES(key.ip_dst), IPv6_BYTES(key.ip_src),
> -	        key.port_dst, key.port_src, key.proto);
> +			"port dst = %d, port src = %d, proto = %d\n",
> +			IPv6_BYTES(key.ip_dst), IPv6_BYTES(key.ip_src),
> +			key.port_dst, key.port_src, key.proto);
>  }
> 

No Change here , so should you undo this change.

>  static inline uint16_t
> @@ -674,9 +692,9 @@ parse_ptype_one(struct rte_mbuf *m)
> 
>  static uint16_t
>  cb_parse_ptype(uint16_t port __rte_unused, uint16_t queue __rte_unused,
> -	       struct rte_mbuf *pkts[], uint16_t nb_pkts,
> -	       uint16_t max_pkts __rte_unused,
> -	       void *user_param __rte_unused)
> +		   struct rte_mbuf *pkts[], uint16_t nb_pkts,
> +		   uint16_t max_pkts __rte_unused,
> +		   void *user_param __rte_unused)

No change here, so should undo this change.

> -			     uint16_t queue_id)
> +				 uint16_t port_id,
> +				 uint16_t queue_id)

No change here, so should undo this change.

>  {
>  	uint32_t rxq_count = rte_eth_rx_queue_count(port_id, queue_id);
>  /**
> @@ -1051,7 +1069,7 @@ static int main_intr_loop(__rte_unused void
> *dummy)
>  			 * less as possible.
>  			 */
>  			for (i = 1,
> -			    lcore_idle_hint = qconf-
> >rx_queue_list[0].idle_hint;
> +				lcore_idle_hint = qconf-
> >rx_queue_list[0].idle_hint;

No change here, so should undo this change.


> @@ -1616,6 +1634,9 @@ print_usage(const char *prgname)
>  		"  [--max-pkt-len PKTLEN]\n"
>  		"  -p PORTMASK: hexadecimal bitmask of ports to
> configure\n"
>  		"  -P: enable promiscuous mode\n"
> +		"  -u: set min frequency for uncore\n"
> +		"  -U: set max frequency for uncore\n"
> +		"  -i (frequency index): set frequency index for uncore\n"

Might be eidt help text a bit, " specify the uncore frequency index that uncore should be set to."

> +	d = opendir(UNCORE_FREQUENCY_DIR);
> +	if (!d) {
> +		RTE_LOG(ERR, EAL, "Unable to open uncore frequency
> directory");

Log is for L3FWD_POWER not EAL. So need to fix this.  d == NULL should be checked rather !d. Also print error string returned by the opendir in the log.

> +		return -1;
> +	}
> +
> +	else {

Else should start in the same line where if ends. Here in other places. 
You don't need else perhaps. 


<snip>

> @@ -1861,10 +1988,12 @@ parse_args(int argc, char **argv)
>  		{CMD_LINE_OPT_SCALE_FREQ_MAX, 1, 0, 0},
>  		{NULL, 0, 0, 0}
>  	};
> +	const char *min = "min";
> +	const char *max = "max";

Instead of using strings to specify if user option is min or max. You can use either of the below options

1) Use global Integer value  and set that to 1 in U case . And set to 0 in u case. 
2)OR have parse_uncore_min_max() function with  one default argument set to 0. When the argument is passed to 1 assume it to be max freq to set else min freq to set.

Thanks,
Reshma

RE: [PATCH v2 3/3] l3fwd-power: add option to call uncore API

[Pattan, Reshma]

> -----Original Message-----
> From: Kearney, Tadhg <tadhg.kearney@intel.com>
<snip>

> +static int
> +parse_uncore_index(const char *choice)
> +{
> +		if (frequency_index > freq_array_len) {

Should this be compared against freq_array_len-1 ?. Ex: If total 10 frequencies the index can be only 0-9.

Thanks,
Reshma

Re: [PATCH v2 3/3] l3fwd-power: add option to call uncore API

[Hunt, David]

Hi Tadhg,

On 13/07/2022 15:07, Tadhg Kearney wrote:
> Add option for setting uncore frequency min/max/index, through uncore api.
> This will be set for each package and die on the SKU. On exit, uncore
> frequency will be reverted back to previous frequency.
>
> Signed-off-by: Tadhg Kearney <tadhg.kearney@intel.com>
> ---
>   .../sample_app_ug/l3_forward_power_man.rst    |  28 +++
>   examples/l3fwd-power/main.c                   | 190 ++++++++++++++++--
>   2 files changed, 202 insertions(+), 16 deletions(-)
>
> diff --git a/doc/guides/sample_app_ug/l3_forward_power_man.rst b/doc/guides/sample_app_ug/l3_forward_power_man.rst
> index 8f6d906200..1e452140a1 100644
> --- a/doc/guides/sample_app_ug/l3_forward_power_man.rst
> +++ b/doc/guides/sample_app_ug/l3_forward_power_man.rst
> @@ -97,6 +97,12 @@ where,
>   *   -P: Sets all ports to promiscuous mode so that packets are accepted regardless of the packet's Ethernet MAC destination address.
>       Without this option, only packets with the Ethernet MAC destination address set to the Ethernet address of the port are accepted.
>   
> +*   -u: optional, sets uncore frequency to minimum value.
> +
> +*   -U: optional, sets uncore frequency to maximum value.
> +
> +*   -i (frequency index): optional, sets uncore frequency to frequency index value, by setting min and max values to be the same.
> +
>   *   --config (port,queue,lcore)[,(port,queue,lcore)]: determines which queues from which ports are mapped to which cores.
>   
>   *   --max-pkt-len: optional, maximum packet length in decimal (64-9600)
> @@ -364,3 +370,25 @@ in the DPDK Programmer's Guide for more details on PMD power management.
>   .. code-block:: console
>   
>           ./<build_dir>/examples/dpdk-l3fwd-power -l 1-3 -- -p 0x0f --config="(0,0,2),(0,1,3)" --pmd-mgmt=scale
> +
> +Setting Uncore Values
> +---------------------
> +
> +Uncore frequency can be adjusted through manipulating related sysfs entries to adjust the minimum and maximum uncore values.
> +This will be set for each package and die on the SKU. Three options are available for setting uncore frequency:
> +
> +``-u``
> +  This will set uncore to minimum frequency possible.
> +
> +``-U``
> +  This will set uncore to maximum frequency possible.
> +
> +``-i``
> +  This will allow you to set the specific uncore frequency index that you want, by setting
> +  minimum and maximum values to be the same. Frequency index's are set 100000Hz apart from
> +  maximum to minimum.
> +  Frequency index values are in descending order, ie, index 0 is maximum frequency index.
> +
> +.. code-block:: console
> +
> +        ./<build_dir>/examples/dpdk-l3fwd-power -l 1-3 -- -p 0x0f --config="(0,0,2),(0,1,3)" -i 1
> diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
> index 887c6eae3f..5f74e29e3a 100644
> --- a/examples/l3fwd-power/main.c
> +++ b/examples/l3fwd-power/main.c
> @@ -15,6 +15,8 @@
>   #include <unistd.h>
>   #include <signal.h>
>   #include <math.h>
> +#include <dirent.h>
> +#include <fnmatch.h>
>   
>   #include <rte_common.h>
>   #include <rte_byteorder.h>
> @@ -47,6 +49,7 @@
>   #include <rte_metrics.h>
>   #include <rte_telemetry.h>
>   #include <rte_power_pmd_mgmt.h>
> +#include <rte_power_uncore.h>
>   
>   #include "perf_core.h"
>   #include "main.h"
> @@ -71,6 +74,7 @@
>   
>   #ifndef APP_LOOKUP_METHOD
>   #define APP_LOOKUP_METHOD             APP_LOOKUP_LPM
> +
>   #endif
>   
>   #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
> @@ -83,7 +87,7 @@
>   
>   #ifndef IPv6_BYTES
>   #define IPv6_BYTES_FMT "%02x%02x:%02x%02x:%02x%02x:%02x%02x:"\
> -                       "%02x%02x:%02x%02x:%02x%02x:%02x%02x"
> +					   "%02x%02x:%02x%02x:%02x%02x:%02x%02x"
>   #define IPv6_BYTES(addr) \
>   	addr[0],  addr[1], addr[2],  addr[3], \
>   	addr[4],  addr[5], addr[6],  addr[7], \
> @@ -134,9 +138,14 @@
>   
>   #define NUM_TELSTATS RTE_DIM(telstats_strings)
>   
> +#define UNCORE_FREQUENCY_DIR "/sys/devices/system/cpu/intel_uncore_frequency"
> +
>   static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
>   static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
>   
> +/* Max number of nodes times dies available on uncore */
> +#define MAX_DIE_NODES (RTE_MAX_NUMA_DIE * RTE_MAX_NUMA_NODES)
> +
>   /* ethernet addresses of ports */
>   static struct rte_ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
>   
> @@ -145,6 +154,8 @@ static rte_spinlock_t locks[RTE_MAX_ETHPORTS];
>   
>   /* mask of enabled ports */
>   static uint32_t enabled_port_mask = 0;
> +/* if uncore frequency was enabled without errors */
> +static int enabled_uncore;
>   /* Ports set in promiscuous mode off by default. */
>   static int promiscuous_on = 0;
>   /* NUMA is enabled by default. */
> @@ -165,6 +176,13 @@ struct telstats_name {
>   	char name[RTE_ETH_XSTATS_NAME_SIZE];
>   };
>   
> +struct uncore_info {
> +	unsigned int pkg;
> +	unsigned int die;
> +};
> +
> +struct uncore_info ui[MAX_DIE_NODES];
> +
>   /* telemetry stats to be reported */
>   const struct telstats_name telstats_strings[] = {
>   	{"empty_poll"},
> @@ -557,9 +575,9 @@ static void
>   print_ipv6_key(struct ipv6_5tuple key)
>   {
>   	printf( "IP dst = " IPv6_BYTES_FMT ", IP src = " IPv6_BYTES_FMT ", "
> -	        "port dst = %d, port src = %d, proto = %d\n",
> -	        IPv6_BYTES(key.ip_dst), IPv6_BYTES(key.ip_src),
> -	        key.port_dst, key.port_src, key.proto);
> +			"port dst = %d, port src = %d, proto = %d\n",
> +			IPv6_BYTES(key.ip_dst), IPv6_BYTES(key.ip_src),
> +			key.port_dst, key.port_src, key.proto);
>   }
>   
>   static inline uint16_t
> @@ -674,9 +692,9 @@ parse_ptype_one(struct rte_mbuf *m)
>   
>   static uint16_t
>   cb_parse_ptype(uint16_t port __rte_unused, uint16_t queue __rte_unused,
> -	       struct rte_mbuf *pkts[], uint16_t nb_pkts,
> -	       uint16_t max_pkts __rte_unused,
> -	       void *user_param __rte_unused)
> +		   struct rte_mbuf *pkts[], uint16_t nb_pkts,
> +		   uint16_t max_pkts __rte_unused,
> +		   void *user_param __rte_unused)
>   {
>   	unsigned int i;
>   
> @@ -797,8 +815,8 @@ power_idle_heuristic(uint32_t zero_rx_packet_count)
>   
>   static inline enum freq_scale_hint_t
>   power_freq_scaleup_heuristic(unsigned lcore_id,
> -			     uint16_t port_id,
> -			     uint16_t queue_id)
> +				 uint16_t port_id,
> +				 uint16_t queue_id)
>   {
>   	uint32_t rxq_count = rte_eth_rx_queue_count(port_id, queue_id);
>   /**
> @@ -1051,7 +1069,7 @@ static int main_intr_loop(__rte_unused void *dummy)
>   			 * less as possible.
>   			 */
>   			for (i = 1,
> -			    lcore_idle_hint = qconf->rx_queue_list[0].idle_hint;
> +				lcore_idle_hint = qconf->rx_queue_list[0].idle_hint;
>   					i < qconf->n_rx_queue; ++i) {
>   				rx_queue = &(qconf->rx_queue_list[i]);
>   				if (rx_queue->idle_hint < lcore_idle_hint)
> @@ -1616,6 +1634,9 @@ print_usage(const char *prgname)
>   		"  [--max-pkt-len PKTLEN]\n"
>   		"  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
>   		"  -P: enable promiscuous mode\n"
> +		"  -u: set min frequency for uncore\n"
> +		"  -U: set max frequency for uncore\n"
> +		"  -i (frequency index): set frequency index for uncore\n"
>   		"  --config (port,queue,lcore): rx queues configuration\n"
>   		"  --high-perf-cores CORELIST: list of high performance cores\n"
>   		"  --perf-config: similar as config, cores specified as indices"
> @@ -1672,6 +1693,112 @@ static int parse_max_pkt_len(const char *pktlen)
>   	return len;
>   }
>   
> +static int
> +parse_uncore_dir(void)
> +{
> +	DIR *d;
> +	struct dirent *dir;
> +	char pkg[3], die[3];
> +	unsigned int count = 0;
> +	static const char filter[] = "*freq_khz*";
> +
> +	d = opendir(UNCORE_FREQUENCY_DIR);
> +	if (!d) {
> +		RTE_LOG(ERR, EAL, "Unable to open uncore frequency directory");
> +		return -1;
> +	}
> +
> +	else {
> +		/* Loop through every file name in uncore frequency dir extracting
> +		 * pkg + die numbers from file names.
> +		 */
> +		while ((dir = readdir(d)) != NULL) {
> +			if (fnmatch(filter, dir->d_name, 0) > 0) {
> +				/* Extract pkg + die numbers from string 'package_XX_die_XX' */
> +				sprintf(pkg, "%c%c", dir->d_name[8], dir->d_name[9]);
> +				sprintf(die, "%c%c", dir->d_name[15], dir->d_name[16]);
> +				ui[count].pkg = atoi(pkg);
> +				ui[count].die = atoi(die);


Prefer to use safer function strtol().

Also, I feel that we should move the sysfs accesses into the new set of 
API functions as well. Currently it feels like most of it is in the 
library, but this part is left in the sample app. How about a couple of 
new APIs to get the max pkg and max die?

Could be rte_power_uncore_get_num_pkgs() and 
rte_power_uncore_get_num_dies(pkg)?

That way all the sysfs accesses are hidden.


> +				count++;
> +			}
> +		}
> +		closedir(d);
> +	}
> +	return(count);
> +}
> +
> +static int
> +parse_uncore_min_max(const char *choice)
> +{
> +	int ret = 0, total_files, i;
> +	unsigned int die, pkg;
> +	total_files = parse_uncore_dir();
> +
> +	if (total_files < 1)
> +		return -1;
> +
> +	for (i = 0; i < total_files; i++) {
> +		die = ui[i].die;
> +		pkg = ui[i].pkg;
> +		rte_power_uncore_init(pkg, die);
> +		if (!strncmp(choice, "max", 3)) {
> +			ret = rte_power_uncore_freq_max(pkg, die);
> +			if (ret == -1)
> +				RTE_LOG(INFO, L3FWD_POWER, "Unable to set max uncore value for pkg %02u die %02u\n"\
> +				, pkg, die);
> +		}
> +
> +		else if (!strncmp(choice, "min", 3)) {
> +			ret = rte_power_uncore_freq_min(pkg, die);
> +			if (ret == -1) {
> +				RTE_LOG(INFO, L3FWD_POWER, "Unable to set min uncore value for pkg %02u die %02u\n"\
> +				, pkg, die);
> +				return ret;
> +			}
> +		} else {
> +			RTE_LOG(INFO, L3FWD_POWER, "Uncore choice provided invalid\n");
> +			return -1;
> +		}
> +	}
> +
> +	RTE_LOG(INFO, L3FWD_POWER, "Successfully set max/min uncore frequency.\n");
> +	return ret;
> +}
> +
> +static int
> +parse_uncore_index(const char *choice)
> +{
> +	int ret = 0, total_files, i;
> +	unsigned int  die, pkg;
> +	total_files = parse_uncore_dir();
> +
> +	if (total_files < 1)
> +		return -1;
> +
> +	for (i = 0; i < total_files; i++) {
> +		die = ui[i].die;
> +		pkg = ui[i].pkg;
> +		rte_power_uncore_init(pkg, die);
> +
> +		int frequency_index = atoi(choice);


Again, prefer to use strtol().


> +		int freq_array_len = rte_power_uncore_get_num_freqs(pkg, die);
> +		if (frequency_index > freq_array_len) {
> +			RTE_LOG(INFO, L3FWD_POWER, \
> +			"Frequency index given out of range, please choose a value from 0 to %d.\n" \
> +			, freq_array_len);
> +			return -1;
> +		}
> +		ret = rte_power_set_uncore_freq(pkg, die, frequency_index);
> +		if (ret == -1) {
> +			RTE_LOG(INFO, L3FWD_POWER, \
> +			"Unable to set uncore index value for pkg %02u die %02u\n", pkg, die);
> +			return ret;
> +		}
> +	}
> +	RTE_LOG(INFO, L3FWD_POWER, "Successfully set freq index for uncore\n");
> +	return ret;
> +}
> +
>   static int
>   parse_portmask(const char *portmask)
>   {
> @@ -1861,10 +1988,12 @@ parse_args(int argc, char **argv)
>   		{CMD_LINE_OPT_SCALE_FREQ_MAX, 1, 0, 0},
>   		{NULL, 0, 0, 0}
>   	};
> +	const char *min = "min";
> +	const char *max = "max";
>   
>   	argvopt = argv;
>   
> -	while ((opt = getopt_long(argc, argvopt, "p:l:m:h:P",
> +	while ((opt = getopt_long(argc, argvopt, "p:l:m:h:P:u U i:",
>   				lgopts, &option_index)) != EOF) {
>   
>   		switch (opt) {
> @@ -1893,6 +2022,30 @@ parse_args(int argc, char **argv)
>   			limit = parse_max_pkt_len(optarg);
>   			freq_tlb[HGH] = limit;
>   			break;
> +		case 'u':
> +			enabled_uncore = parse_uncore_min_max(min);
> +			if (enabled_uncore < 0) {
> +				printf("Unable to reach min uncore value\n");
> +				print_usage(prgname);
> +				return -1;
> +			}
> +			break;
> +		case 'U':
> +			enabled_uncore = parse_uncore_min_max(max);
> +			if (enabled_uncore < 0) {
> +				printf("Unable to reach max uncore value\n");
> +				print_usage(prgname);
> +				return -1;
> +			}
> +			break;
> +		case 'i':
> +			enabled_uncore = parse_uncore_index(optarg);
> +			if (enabled_uncore < 0) {
> +				printf("Unable to reach uncore index value\n");
> +				print_usage(prgname);
> +				return -1;
> +			}
> +			break;
>   		/* long options */
>   		case 0:
>   			if (!strncmp(lgopts[option_index].name, "config", 6)) {
> @@ -2003,8 +2156,8 @@ parse_args(int argc, char **argv)
>   			}
>   
>   			if (!strncmp(lgopts[option_index].name,
> -				     CMD_LINE_OPT_PARSE_PTYPE,
> -				     sizeof(CMD_LINE_OPT_PARSE_PTYPE))) {
> +					 CMD_LINE_OPT_PARSE_PTYPE,
> +					 sizeof(CMD_LINE_OPT_PARSE_PTYPE))) {
>   				printf("soft parse-ptype is enabled\n");
>   				parse_ptype = 1;
>   			}
> @@ -2262,7 +2415,7 @@ check_all_ports_link_status(uint32_t port_mask)
>   				rte_eth_link_to_str(link_status_text,
>   					sizeof(link_status_text), &link);
>   				printf("Port %d %s\n", portid,
> -				       link_status_text);
> +					   link_status_text);
>   				continue;
>   			}
>   			/* clear all_ports_up flag if any link down */
> @@ -2365,6 +2518,8 @@ static int
>   deinit_power_library(void)
>   {
>   	unsigned int lcore_id;
> +	int i, total_files;
> +	total_files = parse_uncore_dir();
>   	int ret = 0;
>   
>   	RTE_LCORE_FOREACH(lcore_id) {
> @@ -2377,6 +2532,9 @@ deinit_power_library(void)
>   			return ret;
>   		}
>   	}
> +
> +	for (i = 0; i < total_files; i++)
> +		rte_power_uncore_exit(ui[i].pkg, ui[i].die);
>   	return ret;
>   }
>   
> @@ -2734,7 +2892,7 @@ main(int argc, char **argv)
>   					"err=%d, port=%d\n", ret, portid);
>   
>   		ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd,
> -						       &nb_txd);
> +							   &nb_txd);
>   		if (ret < 0)
>   			rte_exit(EXIT_FAILURE,
>   				 "Cannot adjust number of descriptors: err=%d, port=%d\n",
> @@ -2791,7 +2949,7 @@ main(int argc, char **argv)
>   			txconf = &dev_info.default_txconf;
>   			txconf->offloads = local_port_conf.txmode.offloads;
>   			ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
> -						     socketid, txconf);
> +							 socketid, txconf);
>   			if (ret < 0)
>   				rte_exit(EXIT_FAILURE,
>   					"rte_eth_tx_queue_setup: err=%d, "

[PATCH v3 0/3] add uncore api to be called through l3fwd-power

[Tadhg Kearney]

This is targeting 22.11 and aims to add an API to DPDK power library to allow uncore frequency adjustment.
This will be called through the l3fwd-power app and gives the ability to set the minimum 
and maximum uncore frequency to both min, max or specific frequency index.

Signed-off-by: tadhgkearney <tadhg.kearney@intel.com>
---

v2:
Fix compilation warnings and errors.
v3:
Remove addition of x86 global macros.
Add 2 new API's for getting package and die numbers from system.
Address comments from mailing list.
Improve efficiency of code and code quality.


Tadhg Kearney (3):
  power: add uncore frequency control API to the power library
  l3fwd-power: add option to call uncore API
  test/power: add unit tests for uncore API

 app/test/meson.build                          |   2 +
 app/test/test_power_uncore.c                  | 293 ++++++++++++
 doc/guides/prog_guide/power_man.rst           |  37 ++
 doc/guides/rel_notes/release_22_11.rst        |   5 +
 .../sample_app_ug/l3_forward_power_man.rst    |  29 ++
 examples/l3fwd-power/main.c                   | 122 ++++-
 lib/power/meson.build                         |   2 +
 lib/power/rte_power_uncore.c                  | 447 ++++++++++++++++++
 lib/power/rte_power_uncore.h                  | 186 ++++++++
 lib/power/version.map                         |  11 +
 10 files changed, 1131 insertions(+), 3 deletions(-)
 create mode 100644 app/test/test_power_uncore.c
 create mode 100644 lib/power/rte_power_uncore.c
 create mode 100644 lib/power/rte_power_uncore.h

-- 
2.25.1

[PATCH v3 1/3] power: add uncore frequency control API to the power library

[Tadhg Kearney]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=y, Size: 21359 bytes --]

Add API to allow uncore frequency adjustment. This is done through
manipulating related uncore frequency control sysfs entries to
adjust the minimum and maximum uncore frequency values.
Nine API's are being added that are all public and experimental.

Signed-off-by: Tadhg Kearney <tadhg.kearney@intel.com>
---
 doc/guides/prog_guide/power_man.rst    |  37 ++
 doc/guides/rel_notes/release_22_11.rst |   5 +
 lib/power/meson.build                  |   2 +
 lib/power/rte_power_uncore.c           | 447 +++++++++++++++++++++++++
 lib/power/rte_power_uncore.h           | 186 ++++++++++
 lib/power/version.map                  |  11 +
 6 files changed, 688 insertions(+)
 create mode 100644 lib/power/rte_power_uncore.c
 create mode 100644 lib/power/rte_power_uncore.h

diff --git a/doc/guides/prog_guide/power_man.rst b/doc/guides/prog_guide/power_man.rst
index 98cfd3c1f3..49ff3edef0 100644
--- a/doc/guides/prog_guide/power_man.rst
+++ b/doc/guides/prog_guide/power_man.rst
@@ -276,6 +276,43 @@ API Overview for Ethernet PMD Power Management
 * **Set Scaling Max Freq**: Set the maximum frequency (kHz) to be used in Frequency
   Scaling mode.
 
+Uncore API
+----------
+
+Abstract
+~~~~~~~~
+
+Uncore is a term used by Intel to describe the functions of a microprocessor that are
+not in the core, but which must be closely connected to the core to achieve high performance;
+L3 cache, on-die memory controller, etc.
+Significant power savings can be achieved by reducing the uncore frequency to its lowest value.
+
+The Linux kernel provides the driver “intel-uncore-frequency" to control the uncore frequency limits
+for x86 platform. The driver is available from kernel version 5.6 and above.
+This manipulates the contest of MSR 0x620, which sets min/max of the uncore for the SKU.
+
+
+API Overview for Uncore
+~~~~~~~~~~~~~~~~~~~~~~~
+* **Uncore Power Init**: Initialise uncore power, populate frequency array and record
+  original min & max for pkg & die.
+
+* **Uncore Power Exit**: Exit uncore power, restoring original min & max for pkg & die.
+
+* **Get Uncore Power Freq**: Get current uncore freq index for pkg & die.
+
+* **Set Uncore Power Freq**: Set min & max uncore freq index for pkg & die (min and max will be the same).
+
+* **Uncore Power Max**: Set max uncore freq index for pkg & die.
+
+* **Uncore Power Min**: Set min uncore freq index for pkg & die.
+
+* **Get Num Freqs**: Get the number of frequencies in the index array.
+
+* **Get Num Pkgs**: Get the number of packages (CPUs) on the system.
+
+* **Get Num Dies**: Get the number of die's on a given package.
+
 References
 ----------
 
diff --git a/doc/guides/rel_notes/release_22_11.rst b/doc/guides/rel_notes/release_22_11.rst
index 8c021cf050..ad09a0acb2 100644
--- a/doc/guides/rel_notes/release_22_11.rst
+++ b/doc/guides/rel_notes/release_22_11.rst
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added uncore frequency control API to the power library.**
+
+  Add api to allow uncore frequency adjustment. This is done through
+  manipulating related uncore frequency control sysfs entries to 
+  adjust the minimum and maximum uncore frequency values.
 
 Removed Items
 -------------
diff --git a/lib/power/meson.build b/lib/power/meson.build
index ba8d66074b..80cdeb72d4 100644
--- a/lib/power/meson.build
+++ b/lib/power/meson.build
@@ -21,12 +21,14 @@ sources = files(
         'rte_power.c',
         'rte_power_empty_poll.c',
         'rte_power_pmd_mgmt.c',
+        'rte_power_uncore.c',
 )
 headers = files(
         'rte_power.h',
         'rte_power_empty_poll.h',
         'rte_power_pmd_mgmt.h',
         'rte_power_guest_channel.h',
+        'rte_power_uncore.h',
 )
 if cc.has_argument('-Wno-cast-qual')
     cflags += '-Wno-cast-qual'
diff --git a/lib/power/rte_power_uncore.c b/lib/power/rte_power_uncore.c
new file mode 100644
index 0000000000..e43a8195dc
--- /dev/null
+++ b/lib/power/rte_power_uncore.c
@@ -0,0 +1,447 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2022 Intel Corporation
+ */
+
+#include <errno.h>
+#include <dirent.h>
+#include <fnmatch.h>
+
+#include <rte_memcpy.h>
+
+#include "rte_power_uncore.h"
+#include "power_common.h"
+
+#define MAX_UNCORE_FREQS 32
+#define MAX_NUMA_DIE 8
+#define BUS_FREQ     100000
+#define FILTER_LENGTH 18
+#define PACKAGE_FILTER "package_%02u_die_*"
+#define DIE_FILTER "package_%02u_die_%02u"
+#define UNCORE_FREQUENCY_DIR "/sys/devices/system/cpu/intel_uncore_frequency"
+#define POWER_GOVERNOR_PERF "performance"
+#define POWER_UNCORE_SYSFILE_MAX_FREQ \
+		"/sys/devices/system/cpu/intel_uncore_frequency/package_%02u_die_%02u/max_freq_khz"
+#define POWER_UNCORE_SYSFILE_MIN_FREQ  \
+		"/sys/devices/system/cpu/intel_uncore_frequency/package_%02u_die_%02u/min_freq_khz"
+#define POWER_UNCORE_SYSFILE_BASE_MAX_FREQ \
+		"/sys/devices/system/cpu/intel_uncore_frequency/package_%02u_die_%02u/initial_max_freq_khz"
+#define POWER_UNCORE_SYSFILE_BASE_MIN_FREQ  \
+		"/sys/devices/system/cpu/intel_uncore_frequency/package_%02u_die_%02u/initial_min_freq_khz"
+
+
+struct uncore_power_info {
+	unsigned int die;                    /** Core die id */
+	unsigned int pkg;                    /** Package id */
+	uint32_t freqs[MAX_UNCORE_FREQS];/** Frequency array */
+	uint32_t nb_freqs;                   /** Number of available freqs */
+	FILE *f_cur_min;                     /** FD of scaling_min */
+	FILE *f_cur_max;                     /** FD of scaling_max */
+	uint32_t curr_idx;                   /** Freq index in freqs array */
+	uint32_t org_min_freq;               /** Original min freq of uncore */
+	uint32_t org_max_freq;               /** Original max freq of uncore */
+	uint32_t init_max_freq;              /** System max uncore freq */
+	uint32_t init_min_freq;              /** System min uncore freq */
+} __rte_cache_aligned;
+
+static struct uncore_power_info uncore_info[RTE_MAX_NUMA_NODES][MAX_NUMA_DIE];
+
+static int
+set_uncore_freq_internal(struct uncore_power_info *ui, uint32_t idx)
+{
+	uint32_t target_uncore_freq, curr_max_freq;
+	int ret;
+
+	if (idx >= MAX_UNCORE_FREQS || idx >= ui->nb_freqs) {
+		RTE_LOG(DEBUG, POWER, "Invalid uncore frequency index %u, which "
+				"should be less than %u\n", idx, ui->nb_freqs);
+		return -1;
+	}
+
+	target_uncore_freq = ui->freqs[idx];
+
+	/* check current max freq, so that the value to be flushed first
+	 * can be accurately recorded
+	 */
+	open_core_sysfs_file(&ui->f_cur_max, "rw+", POWER_UNCORE_SYSFILE_MAX_FREQ,
+			ui->pkg, ui->die);
+	if (ui->f_cur_max == NULL) {
+		RTE_LOG(DEBUG, POWER, "failed to open %s\n",
+				POWER_UNCORE_SYSFILE_MAX_FREQ);
+		return -1;
+	}
+	ret = read_core_sysfs_u32(ui->f_cur_max, &curr_max_freq);
+	if (ret < 0) {
+		RTE_LOG(DEBUG, POWER, "Failed to read %s\n",
+				POWER_UNCORE_SYSFILE_MAX_FREQ);
+		fclose(ui->f_cur_max);
+		return -1;
+	}
+
+	/* check this value first before fprintf value to f_cur_max, so value isn't overwritten */
+	if (fprintf(ui->f_cur_min, "%u", target_uncore_freq) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new uncore frequency for "
+				"pkg %02u die %02u\n", ui->pkg, ui->die);
+		return -1;
+	}
+
+	if (fprintf(ui->f_cur_max, "%u", target_uncore_freq) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new uncore frequency for "
+				"pkg %02u die %02u\n", ui->pkg, ui->die);
+		return -1;
+	}
+
+	POWER_DEBUG_TRACE("Uncore requency '%u' to be set for pkg %02u die %02u\n",
+				target_uncore_freq, ui->pkg, ui->die);
+
+
+	/* write the minimum value first if the target freq is less than current max */
+	if (target_uncore_freq <= curr_max_freq) {
+		fflush(ui->f_cur_min);
+		fflush(ui->f_cur_max);
+	} else {
+		fflush(ui->f_cur_max);
+		fflush(ui->f_cur_min);
+	}
+	ui->curr_idx = idx;
+
+	return 0;
+}
+
+/**
+ * Fopen the sys file for the future setting of the uncore die frequency.
+ */
+static int
+power_init_for_setting_uncore_freq(struct uncore_power_info *ui)
+{
+	FILE *f_base_min = NULL, *f_base_max = NULL, *f_min = NULL, *f_max = NULL;
+	uint32_t base_min_freq, base_max_freq, min_freq, max_freq;
+	int ret;
+
+	/* open and read all uncore sys files */
+	/* Base max */
+	open_core_sysfs_file(&f_base_max, "r", POWER_UNCORE_SYSFILE_BASE_MAX_FREQ,
+			ui->pkg, ui->die);
+	if (f_base_max == NULL) {
+		RTE_LOG(DEBUG, POWER, "failed to open %s\n",
+				POWER_UNCORE_SYSFILE_BASE_MAX_FREQ);
+		goto err;
+	}
+	ret = read_core_sysfs_u32(f_base_max, &base_max_freq);
+	if (ret < 0) {
+		RTE_LOG(DEBUG, POWER, "Failed to read %s\n",
+				POWER_UNCORE_SYSFILE_BASE_MAX_FREQ);
+		goto err;
+	}
+
+	/* Base min */
+	open_core_sysfs_file(&f_base_min, "r", POWER_UNCORE_SYSFILE_BASE_MIN_FREQ,
+		ui->pkg, ui->die);
+	if (f_base_min == NULL) {
+		RTE_LOG(DEBUG, POWER, "failed to open %s\n",
+				POWER_UNCORE_SYSFILE_BASE_MIN_FREQ);
+		goto err;
+	}
+	if (f_base_min != NULL) {
+		ret = read_core_sysfs_u32(f_base_min, &base_min_freq);
+		if (ret < 0) {
+			RTE_LOG(DEBUG, POWER, "Failed to read %s\n",
+					POWER_UNCORE_SYSFILE_BASE_MIN_FREQ);
+			goto err;
+		}
+	}
+
+	/* Curr min */
+	open_core_sysfs_file(&f_min, "rw+", POWER_UNCORE_SYSFILE_MIN_FREQ,
+			ui->pkg, ui->die);
+	if (f_min == NULL) {
+		RTE_LOG(DEBUG, POWER, "failed to open %s\n",
+				POWER_UNCORE_SYSFILE_MIN_FREQ);
+		goto err;
+	}
+	if (f_min != NULL) {
+		ret = read_core_sysfs_u32(f_min, &min_freq);
+		if (ret < 0) {
+			RTE_LOG(DEBUG, POWER, "Failed to read %s\n",
+					POWER_UNCORE_SYSFILE_MIN_FREQ);
+			goto err;
+		}
+	}
+
+	/* Curr max */
+	open_core_sysfs_file(&f_max, "rw+", POWER_UNCORE_SYSFILE_MAX_FREQ,
+			ui->pkg, ui->die);
+	if (f_max == NULL) {
+		RTE_LOG(DEBUG, POWER, "failed to open %s\n",
+				POWER_UNCORE_SYSFILE_MAX_FREQ);
+		goto err;
+	}
+	if (f_max != NULL) {
+		ret = read_core_sysfs_u32(f_max, &max_freq);
+		if (ret < 0) {
+			RTE_LOG(DEBUG, POWER, "Failed to read %s\n",
+					POWER_UNCORE_SYSFILE_MAX_FREQ);
+			goto err;
+		}
+	}
+
+	/* assign file handles */
+	ui->f_cur_min = f_min;
+	ui->f_cur_max = f_max;
+	/* save current min + max freq's so that they can be restored on exit */
+	ui->org_min_freq = min_freq;
+	ui->org_max_freq = max_freq;
+	ui->init_max_freq = base_max_freq;
+	ui->init_min_freq = base_min_freq;
+
+	return 0;
+
+err:
+	if (f_base_min != NULL)
+		fclose(f_base_min);
+	if (f_base_max != NULL)
+		fclose(f_base_max);
+	if (f_min != NULL)
+		fclose(f_min);
+	if (f_max != NULL)
+		fclose(f_max);
+	return -1;
+}
+
+/**
+ * Get the available uncore frequencies of the specific die by reading the
+ * sys file.
+ */
+static int
+power_get_available_uncore_freqs(struct uncore_power_info *ui)
+{
+	int ret = -1;
+	uint32_t i, num_uncore_freqs = 0;
+
+	num_uncore_freqs = (ui->init_max_freq - ui->init_min_freq) / BUS_FREQ + 1;
+	if (num_uncore_freqs >= MAX_UNCORE_FREQS) {
+		RTE_LOG(ERR, POWER, "Too many available uncore frequencies: %d\n",
+				num_uncore_freqs);
+		goto out;
+	}
+
+	/* Generate the uncore freq bucket array. */
+	for (i = 0; i < num_uncore_freqs; i++)
+		ui->freqs[i] = ui->init_max_freq - (i) * BUS_FREQ;
+
+	ui->nb_freqs = num_uncore_freqs;
+
+	ret = 0;
+
+	POWER_DEBUG_TRACE("%d frequency(s) of pkg %02u die %02u are available\n",
+			num_uncore_freqs, ui->pkg, ui->die);
+
+out:
+	return ret;
+}
+
+static int
+check_pkg_die_values(unsigned int pkg, unsigned int die)
+{
+	unsigned int max_pkgs, max_dies;
+	max_pkgs = rte_power_uncore_get_num_pkgs();
+	if (max_pkgs == 0)
+		return -1;
+	if (pkg >= max_pkgs) {
+		RTE_LOG(DEBUG, POWER, "Package number %02u can not exceed %u\n",
+				pkg, max_pkgs);
+		return -1;
+	}
+
+	max_dies = rte_power_uncore_get_num_dies(pkg);
+	if (max_dies == 0)
+		return -1;
+	if (die >= max_dies) {
+		RTE_LOG(DEBUG, POWER, "Die number %02u can not exceed %u\n",
+				die, max_dies);
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+rte_power_uncore_init(unsigned int pkg, unsigned int die)
+{
+	struct uncore_power_info *ui;
+
+	int ret = check_pkg_die_values(pkg, die);
+	if (ret < 0)
+		return -1;
+
+	ui = &uncore_info[pkg][die];
+	ui->die = die;
+	ui->pkg = pkg;
+
+	/* Init for setting uncore die frequency */
+	if (power_init_for_setting_uncore_freq(ui) < 0) {
+		RTE_LOG(DEBUG, POWER, "Cannot init for setting uncore frequency for "
+				"pkg %02u die %02u\n", pkg, die);
+		return -1;
+	}
+
+	/* Get the available frequencies */
+	if (power_get_available_uncore_freqs(ui) < 0) {
+		RTE_LOG(DEBUG, POWER, "Cannot get available uncore frequencies of "
+				"pkg %02u die %02u\n", pkg, die);
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+rte_power_uncore_exit(unsigned int pkg, unsigned int die)
+{
+	struct uncore_power_info *ui;
+
+	int ret = check_pkg_die_values(pkg, die);
+	if (ret < 0)
+		return -1;
+
+	ui = &uncore_info[pkg][die];
+
+	if (fprintf(ui->f_cur_min, "%u", ui->org_min_freq) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write original uncore frequency for "
+				"pkg %02u die %02u\n", ui->pkg, ui->die);
+		return -1;
+	}
+
+	if (fprintf(ui->f_cur_max, "%u", ui->org_max_freq) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write original uncore frequency for "
+				"pkg %02u die %02u\n", ui->pkg, ui->die);
+		return -1;
+	}
+
+	fflush(ui->f_cur_min);
+	fflush(ui->f_cur_max);
+
+	/* Close FD of setting freq */
+	fclose(ui->f_cur_min);
+	fclose(ui->f_cur_max);
+	ui->f_cur_min = NULL;
+	ui->f_cur_max = NULL;
+
+	return 0;
+}
+
+uint32_t
+rte_power_get_uncore_freq(unsigned int pkg, unsigned int die)
+{
+	int ret = check_pkg_die_values(pkg, die);
+	if (ret < 0)
+		return -1;
+
+	return uncore_info[pkg][die].curr_idx;
+}
+
+int
+rte_power_set_uncore_freq(unsigned int pkg, unsigned int die, uint32_t index)
+{
+	int ret = check_pkg_die_values(pkg, die);
+	if (ret < 0)
+		return -1;
+
+	return set_uncore_freq_internal(&(uncore_info[pkg][die]), index);
+}
+
+int
+rte_power_uncore_freq_max(unsigned int pkg, unsigned int die)
+{
+	int ret = check_pkg_die_values(pkg, die);
+	if (ret < 0)
+		return -1;
+
+	return set_uncore_freq_internal(&(uncore_info[pkg][die]), 0);
+}
+
+
+int
+rte_power_uncore_freq_min(unsigned int pkg, unsigned int die)
+{
+	int ret = check_pkg_die_values(pkg, die);
+	if (ret < 0)
+		return -1;
+
+	struct uncore_power_info *ui = &uncore_info[pkg][die];
+
+	return set_uncore_freq_internal(&(uncore_info[pkg][die]), ui->nb_freqs - 1);
+}
+
+int
+rte_power_uncore_get_num_freqs(unsigned int pkg, unsigned int die)
+{
+	int ret = check_pkg_die_values(pkg, die);
+	if (ret < 0)
+		return -1;
+
+	return uncore_info[pkg][die].nb_freqs;
+}
+
+unsigned int
+rte_power_uncore_get_num_pkgs(void)
+{
+	DIR *d;
+	struct dirent *dir;
+	unsigned int count = 0;
+	char filter[FILTER_LENGTH];
+
+	d = opendir(UNCORE_FREQUENCY_DIR);
+	if (d == NULL) {
+		RTE_LOG(ERR, POWER,
+		"Uncore frequency management only supported on x86 with linux kernal >= 5.6.");
+		return 0;
+	}
+
+	/* search by incrementing file name for max pkg file value */
+	while ((dir = readdir(d)) != NULL) {
+		snprintf(filter, FILTER_LENGTH, PACKAGE_FILTER, count);
+		/* make sure filter string is in file name (don't include hidden files) */
+		if (fnmatch(filter, dir->d_name, 0) == 0)
+			count++;
+	}
+
+	closedir(d);
+
+	return count;
+}
+
+unsigned int
+rte_power_uncore_get_num_dies(unsigned int pkg)
+{
+	DIR *d;
+	struct dirent *dir;
+	unsigned int count = 0, max_pkgs;
+	char filter[FILTER_LENGTH];
+
+	max_pkgs = rte_power_uncore_get_num_pkgs();
+	if (max_pkgs == 0)
+		return 0;
+	if (pkg >= max_pkgs) {
+		RTE_LOG(DEBUG, POWER, "Invalid package number\n");
+		return 0;
+	}
+
+	d = opendir(UNCORE_FREQUENCY_DIR);
+	if (d == NULL) {
+		RTE_LOG(ERR, POWER,
+		"Uncore frequency management only supported on x86 with linux kernal >= 5.6.");
+		return 0;
+	}
+
+	/* search by incrementing file name for max die file value */
+	while ((dir = readdir(d)) != NULL) {
+		snprintf(filter, FILTER_LENGTH, DIE_FILTER, pkg, count);
+		/* make sure filter string is in file name (don't include hidden files) */
+		if (fnmatch(filter, dir->d_name, 0) == 0)
+			count++;
+	}
+
+	closedir(d);
+
+	return count;
+}
diff --git a/lib/power/rte_power_uncore.h b/lib/power/rte_power_uncore.h
new file mode 100644
index 0000000000..fac97f97fc
--- /dev/null
+++ b/lib/power/rte_power_uncore.h
@@ -0,0 +1,186 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2022 Intel Corporation
+ */
+
+#ifndef _RTE_POWER_UNCORE_H
+#define _RTE_POWER_UNCORE_H
+
+/**
+ * @file
+ * RTE Uncore Frequency Management
+ */
+
+#include "rte_power.h"
+
+/**
+ * Initialize uncore frequency management for specific die on a package. It will get the available
+ * frequencies and prepare to set new die frequencies.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+__rte_experimental
+int
+rte_power_uncore_init(unsigned int pkg, unsigned int die);
+
+/**
+ * Exit uncore frequency management on a specific die on a package. It will restore uncore min and
+ * max values to previous values before initialization of API.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+__rte_experimental
+int
+rte_power_uncore_exit(unsigned int pkg, unsigned int die);
+
+/**
+ * Return the current index of available frequencies of a specific die on a package.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ *
+ * @return
+ *  The current index of available frequencies.
+ *  If error, it will return 'RTE_POWER_INVALID_FREQ_INDEX = (~0)'.
+ */
+__rte_experimental
+uint32_t
+rte_power_get_uncore_freq(unsigned int pkg, unsigned int die);
+
+/**
+ * Set the new frequency for a specific die on a package by indicating the index of
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ * @param index
+ *  The index of available frequencies.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+__rte_experimental
+int
+rte_power_set_uncore_freq(unsigned int pkg, unsigned int die, uint32_t index);
+
+/**
+ * Scale up the frequency of a specific die on a package to the highest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+__rte_experimental
+int
+rte_power_uncore_freq_max(unsigned int pkg, unsigned int die);
+
+/**
+ * Scale down the frequency of a specific die on a package to the lowest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+__rte_experimental
+int
+rte_power_uncore_freq_min(unsigned int pkg, unsigned int die);
+
+/**
+ * Return the list length of available frequencies in the index array.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ * @param die
+ *  Die number.
+ *
+ * @return
+ *  - The number of available index's in frequency array.
+ *  - Negative on error.
+ */
+__rte_experimental
+int
+rte_power_uncore_get_num_freqs(unsigned int pkg, unsigned int die);
+
+/**
+ * Return the number of packages (CPUs) on a system by parsing the uncore
+ * sysfs directory.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @return
+ *  - Zero on error.
+ *  - Number of package on system on success.
+ */
+__rte_experimental
+unsigned int
+rte_power_uncore_get_num_pkgs(void);
+
+/**
+ * Return the number of dies for pakckages (CPUs) specified from parsing
+ * the uncore sysfs directory.
+ *
+ * This function should NOT be called in the fast path.
+ *
+ * @param pkg
+ *  Package number.
+ *
+ * @return
+ *  - Zero on error.
+ *  - Number of dies for package on sucecss.
+ */
+__rte_experimental
+unsigned int
+rte_power_uncore_get_num_dies(unsigned int pkg);
+
+#endif
diff --git a/lib/power/version.map b/lib/power/version.map
index f9b2947adf..8fccbf20f7 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -48,4 +48,15 @@ EXPERIMENTAL {
 	rte_power_pmd_mgmt_set_pause_duration;
 	rte_power_pmd_mgmt_set_scaling_freq_max;
 	rte_power_pmd_mgmt_set_scaling_freq_min;
+
+	# added in 22.11
+	rte_power_get_uncore_freq;
+	rte_power_set_uncore_freq;
+	rte_power_uncore_exit;
+	rte_power_uncore_freq_max;
+	rte_power_uncore_freq_min;
+	rte_power_uncore_get_num_dies;
+	rte_power_uncore_get_num_freqs;
+	rte_power_uncore_get_num_pkgs;
+	rte_power_uncore_init;
 };
-- 
2.25.1

[PATCH v3 2/3] l3fwd-power: add option to call uncore API

[Tadhg Kearney]

Add option for setting uncore frequency min/max/index, through uncore API.
This will be set for each package and die on the SKU. On exit, uncore min
and max frequency will be reverted back to previous frequencies.

Signed-off-by: Tadhg Kearney <tadhg.kearney@intel.com>
---
 .../sample_app_ug/l3_forward_power_man.rst    |  29 +++++
 examples/l3fwd-power/main.c                   | 122 +++++++++++++++++-
 2 files changed, 148 insertions(+), 3 deletions(-)

diff --git a/doc/guides/sample_app_ug/l3_forward_power_man.rst b/doc/guides/sample_app_ug/l3_forward_power_man.rst
index 8f6d906200..08ac8ef369 100644
--- a/doc/guides/sample_app_ug/l3_forward_power_man.rst
+++ b/doc/guides/sample_app_ug/l3_forward_power_man.rst
@@ -97,6 +97,12 @@ where,
 *   -P: Sets all ports to promiscuous mode so that packets are accepted regardless of the packet's Ethernet MAC destination address.
     Without this option, only packets with the Ethernet MAC destination address set to the Ethernet address of the port are accepted.
 
+*   -u: optional, sets uncore min/max frequency to minimum value.
+
+*   -U: optional, sets uncore min/max frequency to maximum value.
+
+*   -i (frequency index): optional, sets uncore frequency to frequency index value, by setting min and max values to be the same.
+
 *   --config (port,queue,lcore)[,(port,queue,lcore)]: determines which queues from which ports are mapped to which cores.
 
 *   --max-pkt-len: optional, maximum packet length in decimal (64-9600)
@@ -364,3 +370,26 @@ in the DPDK Programmer's Guide for more details on PMD power management.
 .. code-block:: console
 
         ./<build_dir>/examples/dpdk-l3fwd-power -l 1-3 -- -p 0x0f --config="(0,0,2),(0,1,3)" --pmd-mgmt=scale
+
+Setting Uncore Values
+---------------------
+
+Uncore frequency can be adjusted through manipulating related sysfs entries to adjust the minimum and maximum uncore values.
+This will be set for each package and die on the SKU. The driver for enabling this is available from kernel version 5.6 and above.
+Three options are available for setting uncore frequency:
+
+``-u``
+  This will set uncore minimum and maximum frequencies to minimum possible value.
+
+``-U``
+  This will set uncore minimum and maximum frequencies to maximum possible value.
+
+``-i``
+  This will allow you to set the specific uncore frequency index that you want, by setting
+  the uncore frequency to a frequency pointed by index. Frequency index's are set 100MHz apart from
+  maximum to minimum.
+  Frequency index values are in descending order, ie, index 0 is maximum frequency index.
+
+.. code-block:: console
+
+        ./<build_dir>/examples/dpdk-l3fwd-power -l 1-3 -- -p 0x0f --config="(0,0,2),(0,1,3)" -i 1
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 887c6eae3f..716dd296ff 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -47,6 +47,7 @@
 #include <rte_metrics.h>
 #include <rte_telemetry.h>
 #include <rte_power_pmd_mgmt.h>
+#include <rte_power_uncore.h>
 
 #include "perf_core.h"
 #include "main.h"
@@ -179,6 +180,12 @@ enum busy_rate {
 	FULL = 100
 };
 
+enum uncore_choice {
+	UNCORE_MIN = 0,
+	UNCORE_MAX = 1,
+	UNCORE_IDX = 2
+};
+
 /* reference poll count to measure core busyness */
 #define DEFAULT_COUNT 10000
 /*
@@ -1616,6 +1623,9 @@ print_usage(const char *prgname)
 		"  [--max-pkt-len PKTLEN]\n"
 		"  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
 		"  -P: enable promiscuous mode\n"
+		"  -u: set min/max frequency for uncore to minimum value\n"
+		"  -U: set min/max frequency for uncore to maximum value\n"
+		"  -i (frequency index): set min/max frequency for uncore to specified frequency index\n"
 		"  --config (port,queue,lcore): rx queues configuration\n"
 		"  --high-perf-cores CORELIST: list of high performance cores\n"
 		"  --perf-config: similar as config, cores specified as indices"
@@ -1672,6 +1682,74 @@ static int parse_max_pkt_len(const char *pktlen)
 	return len;
 }
 
+static int
+parse_uncore_options(enum uncore_choice choice, const char *argument)
+{
+	unsigned int die, pkg, max_pkg, max_die;
+	int ret = 0;
+	max_pkg = rte_power_uncore_get_num_pkgs();
+	if (max_pkg == 0)
+		return -1;
+
+	for (pkg = 0; pkg < max_pkg; pkg++) {
+		max_die = rte_power_uncore_get_num_dies(pkg);
+		if (max_die == 0)
+			return -1;
+		for (die = 0; die < max_die; die++) {
+			ret = rte_power_uncore_init(pkg, die);
+			if (ret == -1) {
+				RTE_LOG(INFO, L3FWD_POWER, "Unable to initialize uncore for pkg %02u die %02u\n"
+				, pkg, die);
+				return ret;
+			}
+			if (choice == UNCORE_MIN) {
+				ret = rte_power_uncore_freq_min(pkg, die);
+				if (ret == -1) {
+					RTE_LOG(INFO, L3FWD_POWER,
+					"Unable to set the uncore min/max to minimum uncore frequency value for pkg %02u die %02u\n"
+					, pkg, die);
+					return ret;
+				}
+			} else if (choice == UNCORE_MAX) {
+				ret = rte_power_uncore_freq_max(pkg, die);
+				if (ret == -1) {
+					RTE_LOG(INFO, L3FWD_POWER,
+					"Unable to set uncore min/max to maximum uncore frequency value for pkg %02u die %02u\n"
+					, pkg, die);
+					return ret;
+				}
+			} else if (choice == UNCORE_IDX) {
+				char *ptr = NULL;
+				int frequency_index = strtol(argument, &ptr, 10);
+				if (argument == ptr) {
+					RTE_LOG(INFO, L3FWD_POWER, "Index given is not a valid number.");
+					return -1;
+				}
+				int freq_array_len = rte_power_uncore_get_num_freqs(pkg, die);
+				if (frequency_index > freq_array_len - 1) {
+					RTE_LOG(INFO, L3FWD_POWER,
+					"Frequency index given out of range, please choose a value from 0 to %d.\n",
+					freq_array_len);
+					return -1;
+				}
+				ret = rte_power_set_uncore_freq(pkg, die, frequency_index);
+				if (ret == -1) {
+					RTE_LOG(INFO, L3FWD_POWER,
+					"Unable to set min/max uncore index value for pkg %02u die %02u\n",
+					pkg, die);
+					return ret;
+				}
+			} else {
+				RTE_LOG(INFO, L3FWD_POWER, "Uncore choice provided invalid\n");
+				return -1;
+			}
+		}
+	}
+
+	RTE_LOG(INFO, L3FWD_POWER, "Successfully set max/min/index uncore frequency.\n");
+	return ret;
+}
+
 static int
 parse_portmask(const char *portmask)
 {
@@ -1838,7 +1916,7 @@ parse_ep_config(const char *q_arg)
 static int
 parse_args(int argc, char **argv)
 {
-	int opt, ret;
+	int opt, ret, enabled_uncore = 0;
 	char **argvopt;
 	int option_index;
 	uint32_t limit;
@@ -1864,7 +1942,7 @@ parse_args(int argc, char **argv)
 
 	argvopt = argv;
 
-	while ((opt = getopt_long(argc, argvopt, "p:l:m:h:P",
+	while ((opt = getopt_long(argc, argvopt, "p:l:m:h:P:uUi:",
 				lgopts, &option_index)) != EOF) {
 
 		switch (opt) {
@@ -1893,6 +1971,27 @@ parse_args(int argc, char **argv)
 			limit = parse_max_pkt_len(optarg);
 			freq_tlb[HGH] = limit;
 			break;
+		case 'u':
+			enabled_uncore = parse_uncore_options(UNCORE_MIN, NULL);
+			if (enabled_uncore < 0) {
+				print_usage(prgname);
+				return -1;
+			}
+			break;
+		case 'U':
+			enabled_uncore = parse_uncore_options(UNCORE_MAX, NULL);
+			if (enabled_uncore < 0) {
+				print_usage(prgname);
+				return -1;
+			}
+			break;
+		case 'i':
+			enabled_uncore = parse_uncore_options(UNCORE_IDX, optarg);
+			if (enabled_uncore < 0) {
+				print_usage(prgname);
+				return -1;
+			}
+			break;
 		/* long options */
 		case 0:
 			if (!strncmp(lgopts[option_index].name, "config", 6)) {
@@ -2364,7 +2463,7 @@ init_power_library(void)
 static int
 deinit_power_library(void)
 {
-	unsigned int lcore_id;
+	unsigned int lcore_id, max_pkg, max_die, die, pkg;
 	int ret = 0;
 
 	RTE_LCORE_FOREACH(lcore_id) {
@@ -2377,6 +2476,23 @@ deinit_power_library(void)
 			return ret;
 		}
 	}
+
+	max_pkg = rte_power_uncore_get_num_pkgs();
+	if (max_pkg == 0)
+		return -1;
+	for (pkg = 0; pkg < max_die; pkg++) {
+		max_die = rte_power_uncore_get_num_dies(pkg);
+		if (max_die == 0)
+			return -1;
+		for (die = 0; die < max_die; die++) {
+			ret = rte_power_uncore_exit(pkg, die);
+			if (ret < 0) {
+				RTE_LOG(ERR, L3FWD_POWER, "Failed to exit uncore deninit successfully for pkg %02u die %02u\n"
+					, pkg, die);
+				return -1;
+			}
+		}
+	}
 	return ret;
 }
 
-- 
2.25.1

[PATCH v3 3/3] test/power: add unit tests for uncore API

[Tadhg Kearney]

Add basic unit tests covering all nine uncore API's.

Signed-off-by: Tadhg Kearney <tadhg.kearney@intel.com>
---
 app/test/meson.build         |   2 +
 app/test/test_power_uncore.c | 293 +++++++++++++++++++++++++++++++++++
 2 files changed, 295 insertions(+)
 create mode 100644 app/test/test_power_uncore.c

diff --git a/app/test/meson.build b/app/test/meson.build
index bf1d81f84a..170401ccdc 100644
--- a/app/test/meson.build
+++ b/app/test/meson.build
@@ -100,6 +100,7 @@ test_sources = files(
         'test_power.c',
         'test_power_cpufreq.c',
         'test_power_kvm_vm.c',
+        'test_power_uncore.c',
         'test_prefetch.c',
         'test_rand_perf.c',
         'test_rawdev.c',
@@ -240,6 +241,7 @@ fast_tests = [
         ['power_cpufreq_autotest', false, true],
         ['power_autotest', true, true],
         ['power_kvm_vm_autotest', false, true],
+        ['power_uncore_autotest', true, true],
         ['reorder_autotest', true, true],
         ['service_autotest', true, true],
         ['thash_autotest', true, true],
diff --git a/app/test/test_power_uncore.c b/app/test/test_power_uncore.c
new file mode 100644
index 0000000000..77e7dbc7ca
--- /dev/null
+++ b/app/test/test_power_uncore.c
@@ -0,0 +1,293 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2022 Intel Corporation
+ */
+
+#include "test.h"
+
+#ifndef RTE_LIB_POWER
+
+static int
+test_power_uncore(void)
+{
+	printf("Power management library not supported, skipping test\n");
+	return TEST_SKIPPED;
+}
+
+#else
+#include <rte_power_uncore.h>
+#include <power_common.h>
+
+#define MAX_UNCORE_FREQS 32
+
+#define VALID_PKG 0
+#define VALID_DIE 0
+#define INVALID_PKG (rte_power_uncore_get_num_pkgs() + 1)
+#define INVALID_DIE (rte_power_uncore_get_num_dies(VALID_PKG) + 1)
+#define VALID_INDEX 1
+#define INVALID_INDEX (MAX_UNCORE_FREQS + 1)
+
+static int check_power_uncore_init(void)
+{
+	int ret;
+
+	/* Test initialisation of uncore configuration*/
+	ret = rte_power_uncore_init(VALID_PKG, VALID_DIE);
+	if (ret < 0) {
+		printf("Cannot initialise uncore power management for pkg %u die %u, this "
+			"may occur if environment is not configured "
+			"correctly(APCI cpufreq) or operating in another valid "
+			"Power management environment\n", VALID_PKG, VALID_DIE);
+		return -1;
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_uncore_init(INVALID_PKG, INVALID_DIE);
+	if (ret == 0) {
+		printf("Unexpectedly was able to initialise uncore power management "
+			"for pkg %u die %u\n", INVALID_PKG, INVALID_DIE);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_get_uncore_freq(void)
+{
+	int ret;
+
+	/* Successfully get uncore freq */
+	ret = rte_power_get_uncore_freq(VALID_PKG, VALID_DIE);
+	if (ret < 0) {
+		printf("Failed to get uncore frequency for pkg %u die %u\n",
+							VALID_PKG, VALID_DIE);
+		return -1;
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_get_uncore_freq(INVALID_PKG, INVALID_DIE);
+	if (ret >= 0) {
+		printf("Unexpectedly got invalid uncore frequency for pkg %u die %u\n",
+							INVALID_PKG, INVALID_DIE);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_set_uncore_freq(void)
+{
+	int ret;
+
+	/* Successfully set uncore freq */
+	ret = rte_power_set_uncore_freq(VALID_PKG, VALID_DIE, VALID_INDEX);
+	if (ret < 0) {
+		printf("Failed to set uncore frequency for pkg %u die %u index %u\n",
+							VALID_PKG, VALID_DIE, VALID_INDEX);
+		return -1;
+	}
+
+	/* Try to unsuccesfully set invlaid uncore freq index */
+	ret = rte_power_set_uncore_freq(VALID_PKG, VALID_DIE, INVALID_INDEX);
+	if (ret == 0) {
+		printf("Unexpectedly set invalid uncore index for pkg %u die %u index %u\n",
+							VALID_PKG, VALID_DIE, INVALID_INDEX);
+		return -1;
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_set_uncore_freq(INVALID_PKG, INVALID_DIE, VALID_INDEX);
+	if (ret == 0) {
+		printf("Unexpectedly set invalid uncore frequency for pkg %u die %u index %u\n",
+							INVALID_PKG, INVALID_DIE, VALID_INDEX);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_uncore_freq_max(void)
+{
+	int ret;
+
+	/* Successfully get max uncore freq */
+	ret = rte_power_uncore_freq_max(VALID_PKG, VALID_DIE);
+	if (ret < 0) {
+		printf("Failed to set max uncore frequency for pkg %u die %u\n",
+							VALID_PKG, VALID_DIE);
+		return -1;
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_uncore_freq_max(INVALID_PKG, INVALID_DIE);
+	if (ret == 0) {
+		printf("Unexpectedly set invalid max uncore frequency for pkg %u die %u\n",
+							INVALID_PKG, INVALID_DIE);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_uncore_freq_min(void)
+{
+	int ret;
+
+	/* Successfully get min uncore freq */
+	ret = rte_power_uncore_freq_min(VALID_PKG, VALID_DIE);
+	if (ret < 0) {
+		printf("Failed to set min uncore frequency for pkg %u die %u\n",
+							VALID_PKG, VALID_DIE);
+		return -1;
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_uncore_freq_min(INVALID_PKG, INVALID_DIE);
+	if (ret == 0) {
+		printf("Unexpectedly set invalid min uncore frequency for pkg %u die %u\n",
+							INVALID_PKG, INVALID_DIE);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_uncore_get_num_freqs(void)
+{
+	int ret;
+
+	/* Successfully get number of uncore freq */
+	ret = rte_power_uncore_get_num_freqs(VALID_PKG, VALID_DIE);
+	if (ret < 0) {
+		printf("Failed to get number of uncore frequencies for pkg %u die %u\n",
+							VALID_PKG, VALID_DIE);
+		return -1;
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_uncore_get_num_freqs(INVALID_PKG, INVALID_DIE);
+	if (ret >= 0) {
+		printf("Unexpectedly got number of invalid frequencies for pkg %u die %u\n",
+							INVALID_PKG, INVALID_DIE);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_uncore_get_num_pkgs(void)
+{
+	int ret;
+
+	/* Successfully get number of uncore pkgs */
+	ret = rte_power_uncore_get_num_pkgs();
+	if (ret == 0) {
+		printf("Failed to get number of uncore pkgs\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_uncore_get_num_dies(void)
+{
+	int ret;
+
+	/* Successfully get number of uncore dies */
+	ret = rte_power_uncore_get_num_dies(VALID_PKG);
+	if (ret == 0) {
+		printf("Failed to get number of uncore dies for pkg %u\n",
+							VALID_PKG);
+		return -1;
+	}
+
+	/* Unsuccessful test */
+	ret = rte_power_uncore_get_num_dies(INVALID_PKG);
+	if (ret > 0) {
+		printf("Unexpectedly got number of invalid dies for pkg %u\n",
+							INVALID_PKG);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+check_power_uncore_exit(void)
+{
+	int ret;
+
+	/* Successfully exit uncore power management */
+	ret = rte_power_uncore_exit(VALID_PKG, VALID_DIE);
+	if (ret < 0) {
+		printf("Failed to exit uncore power management for pkg %u die %u\n",
+							VALID_PKG, VALID_DIE);
+	}
+
+	/* Unsuccessful Test */
+	ret = rte_power_uncore_exit(INVALID_PKG, INVALID_DIE);
+	if (ret == 0) {
+		printf("Unexpectedly was able to exit uncore power management for pkg %u die %u\n",
+							INVALID_PKG, INVALID_DIE);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+test_power_uncore(void)
+{
+	int ret;
+
+	ret = check_power_uncore_init();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_get_uncore_freq();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_set_uncore_freq();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_uncore_freq_max();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_uncore_freq_min();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_uncore_get_num_freqs();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_uncore_get_num_pkgs();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_uncore_get_num_dies();
+	if (ret < 0)
+		goto fail_all;
+
+	ret = check_power_uncore_exit();
+	if (ret < 0)
+		return -1;
+
+	return 0;
+
+fail_all:
+	rte_power_uncore_exit(VALID_PKG, VALID_DIE);
+	return -1;
+}
+#endif
+
+REGISTER_TEST_COMMAND(power_uncore_autotest, test_power_uncore);
-- 
2.25.1