[dpdk-dev] [PATCH v1 0/1] power: add support for cppc cpufreq driver

[dpdk-dev] [PATCH v1 0/1] power: add support for cppc cpufreq driver

[Richael Zhuang]

v1:
add support for cppc cpufreq driver

Richael Zhuang (1):
  power: add support for cppc cpufreq

 app/test/test_power.c          |   3 +-
 app/test/test_power_cpufreq.c  |   3 +-
 lib/power/meson.build          |   1 +
 lib/power/power_cppc_cpufreq.c | 807 +++++++++++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h | 229 ++++++++++
 lib/power/rte_power.c          |  26 ++
 lib/power/rte_power.h          |   2 +-
 7 files changed, 1068 insertions(+), 3 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

-- 
2.20.1

[dpdk-dev] [PATCH v1 1/1] power: add support for cppc cpufreq

[Richael Zhuang]

Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
supported, which are both not available on arm64 platforms. Add
support for cppc_cpufreq driver which works on most arm64 platforms.

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
---
 app/test/test_power.c          |   3 +-
 app/test/test_power_cpufreq.c  |   3 +-
 lib/power/meson.build          |   1 +
 lib/power/power_cppc_cpufreq.c | 807 +++++++++++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h | 229 ++++++++++
 lib/power/rte_power.c          |  26 ++
 lib/power/rte_power.h          |   2 +-
 7 files changed, 1068 insertions(+), 3 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

diff --git a/app/test/test_power.c b/app/test/test_power.c
index da1d67c0a..b7b556134 100644
--- a/app/test/test_power.c
+++ b/app/test/test_power.c
@@ -133,7 +133,8 @@ test_power(void)
 	/* Perform tests for valid environments.*/
 	const enum power_management_env envs[] = {PM_ENV_ACPI_CPUFREQ,
 			PM_ENV_KVM_VM,
-			PM_ENV_PSTATE_CPUFREQ};
+			PM_ENV_PSTATE_CPUFREQ,
+			PM_ENV_CPPC_CPUFREQ};
 
 	unsigned int i;
 	for (i = 0; i < RTE_DIM(envs); ++i) {
diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index f753d24ac..87d07191d 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -470,7 +470,8 @@ test_power_cpufreq(void)
 
 	/* Test environment configuration */
 	env = rte_power_get_env();
-	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ)) {
+	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ) &&
+			(env != PM_ENV_CPPC_CPUFREQ)) {
 		printf("Unexpectedly got an environment other than ACPI/PSTATE\n");
 		goto fail_all;
 	}
diff --git a/lib/power/meson.build b/lib/power/meson.build
index a2cc9fe2e..b22cdccc1 100644
--- a/lib/power/meson.build
+++ b/lib/power/meson.build
@@ -14,6 +14,7 @@ sources = files(
         'rte_power.c',
         'rte_power_empty_poll.c',
         'rte_power_pmd_mgmt.c',
+        'power_cppc_cpufreq.c'
 )
 headers = files(
         'rte_power.h',
diff --git a/lib/power/power_cppc_cpufreq.c b/lib/power/power_cppc_cpufreq.c
new file mode 100644
index 000000000..9249baf6e
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.c
@@ -0,0 +1,807 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#include <rte_memcpy.h>
+#include <rte_memory.h>
+
+#include "power_cppc_cpufreq.h"
+#include "power_common.h"
+
+#ifdef RTE_LIBRTE_POWER_DEBUG
+#define POWER_DEBUG_TRACE(fmt, args...) do { \
+		RTE_LOG(ERR, POWER, "%s: " fmt, __func__, ## args); \
+} while (0)
+#else
+#define POWER_DEBUG_TRACE(fmt, args...)
+#endif
+
+#define FOPEN_OR_ERR_RET(f, retval) do { \
+		if ((f) == NULL) { \
+			RTE_LOG(ERR, POWER, "File not opened\n"); \
+			return retval; \
+		} \
+} while (0)
+
+#define FOPS_OR_NULL_GOTO(ret, label) do { \
+		if ((ret) == NULL) { \
+			RTE_LOG(ERR, POWER, "fgets returns nothing\n"); \
+			goto label; \
+		} \
+} while (0)
+
+#define FOPS_OR_ERR_GOTO(ret, label) do { \
+		if ((ret) < 0) { \
+			RTE_LOG(ERR, POWER, "File operations failed\n"); \
+			goto label; \
+		} \
+} while (0)
+
+
+/* macros used for rounding frequency to nearest 100000 */
+#define FREQ_ROUNDING_DELTA 50000
+#define ROUND_FREQ_TO_N_100000 100000
+
+/* the unit of highest_perf and nominal_perf differs on different arm platforms.
+ * For highest_perf, it maybe 300 or 3000000, both means 3.0GHz.
+ */
+#define UNIT_DIFF 10000
+
+#define POWER_CONVERT_TO_DECIMAL 10
+
+#define POWER_GOVERNOR_USERSPACE "userspace"
+#define POWER_SYSFILE_GOVERNOR   \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_governor"
+#define POWER_SYSFILE_SETSPEED   \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_setspeed"
+#define POWER_SYSFILE_SCALING_MAX_FREQ \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
+#define POWER_SYSFILE_SCALING_MIN_FREQ  \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
+#define POWER_SYSFILE_HIGHEST_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/highest_perf"
+#define POWER_SYSFILE_NOMINAL_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/nominal_perf"
+#define POWER_SYSFILE_SYS_MAX \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
+
+#define POWER_CPPC_DRIVER "cppc-cpufreq"
+#define BUS_FREQ     100000
+
+enum power_state {
+	POWER_IDLE = 0,
+	POWER_ONGOING,
+	POWER_USED,
+	POWER_UNKNOWN
+};
+
+/**
+ * Power info per lcore.
+ */
+struct cppc_power_info {
+	unsigned int lcore_id;                   /**< Logical core id */
+	uint32_t state;                      /**< Power in use state */
+	FILE *f;                             /**< FD of scaling_setspeed */
+	char governor_ori[32];               /**< Original governor name */
+	uint32_t curr_idx;                   /**< Freq index in freqs array */
+	uint32_t highest_perf;		     /**< system wide max freq */
+	uint32_t nominal_perf;		     /**< system wide nominal freq */
+	uint16_t turbo_available;            /**< Turbo Boost available */
+	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
+	uint32_t nb_freqs;                   /**< number of available freqs */
+	uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
+} __rte_cache_aligned;
+
+static struct cppc_power_info lcore_power_info[RTE_MAX_LCORE];
+
+/**
+ * It is to set specific freq for specific logical core, according to the index
+ * of supported frequencies.
+ */
+static int
+set_freq_internal(struct cppc_power_info *pi, uint32_t idx)
+{
+	if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
+				"should be less than %u\n", idx, pi->nb_freqs);
+		return -1;
+	}
+
+	/* Check if it is the same as current */
+	if (idx == pi->curr_idx)
+		return 0;
+
+	POWER_DEBUG_TRACE("Frequency[%u] %u to be set for lcore %u\n",
+			idx, pi->freqs[idx], pi->lcore_id);
+	if (fseek(pi->f, 0, SEEK_SET) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
+				"for setting frequency for lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	if (fprintf(pi->f, "%u", pi->freqs[idx]) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new frequency for "
+				"lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	fflush(pi->f);
+	pi->curr_idx = idx;
+
+	return 1;
+}
+
+/**
+ * It is to check the current scaling governor by reading sys file, and then
+ * set it into 'userspace' if it is not by writing the sys file. The original
+ * governor will be saved for rolling back.
+ */
+static int
+power_set_governor_userspace(struct cppc_power_info *pi)
+{
+	FILE *f;
+	int ret = -1;
+	char buf[BUFSIZ];
+	char fullpath[PATH_MAX];
+	char *s;
+	int val;
+
+	snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
+			pi->lcore_id);
+	f = fopen(fullpath, "rw+");
+	FOPEN_OR_ERR_RET(f, ret);
+
+	s = fgets(buf, sizeof(buf), f);
+	FOPS_OR_NULL_GOTO(s, out);
+	/* Strip off terminating '\n' */
+	strtok(buf, "\n");
+
+	/* Check if current governor is userspace */
+	if (strncmp(buf, POWER_GOVERNOR_USERSPACE,
+			sizeof(POWER_GOVERNOR_USERSPACE)) == 0) {
+		ret = 0;
+		POWER_DEBUG_TRACE("Power management governor of lcore %u is "
+				"already userspace\n", pi->lcore_id);
+		goto out;
+	}
+	/* Save the original governor */
+	strlcpy(pi->governor_ori, buf, sizeof(pi->governor_ori));
+
+	/* Write 'userspace' to the governor */
+	val = fseek(f, 0, SEEK_SET);
+	FOPS_OR_ERR_GOTO(val, out);
+
+	val = fputs(POWER_GOVERNOR_USERSPACE, f);
+	FOPS_OR_ERR_GOTO(val, out);
+
+	/* We need to flush to see if the fputs succeeds */
+	val = fflush(f);
+	FOPS_OR_ERR_GOTO(val, out);
+
+	ret = 0;
+	RTE_LOG(INFO, POWER, "Power management governor of lcore %u has been "
+			"set to user space successfully\n", pi->lcore_id);
+out:
+	fclose(f);
+
+	return ret;
+}
+
+static int
+power_check_turbo(struct cppc_power_info *pi)
+{
+	FILE *f_nom, *f_max, *f_cmax;
+	int ret = -1;
+	char *p_nom, *p_max, *p_cmax;
+	char buf_nom[BUFSIZ];
+	char buf_max[BUFSIZ];
+	char buf_cmax[BUFSIZ];
+	char fullpath_nom[PATH_MAX];
+	char fullpath_max[PATH_MAX];
+	char fullpath_cmax[PATH_MAX];
+	char *s_nom, *s_max, *s_cmax;
+	uint32_t nominal_perf = 0, highest_perf = 0, cpuinfo_max_freq = 0;
+
+	snprintf(fullpath_max, sizeof(fullpath_max),
+			POWER_SYSFILE_HIGHEST_PERF,
+			pi->lcore_id);
+	snprintf(fullpath_nom, sizeof(fullpath_nom),
+			POWER_SYSFILE_NOMINAL_PERF,
+			pi->lcore_id);
+	snprintf(fullpath_cmax, sizeof(fullpath_cmax),
+			POWER_SYSFILE_SYS_MAX,
+			pi->lcore_id);
+
+	f_nom = fopen(fullpath_nom, "r");
+	FOPEN_OR_ERR_RET(f_nom, ret);
+
+	f_max = fopen(fullpath_max, "r");
+	if (f_max == NULL)
+		fclose(f_nom);
+	FOPEN_OR_ERR_RET(f_max, ret);
+
+	f_cmax = fopen(fullpath_cmax, "r");
+	if (f_cmax == NULL) {
+		fclose(f_max);
+		fclose(f_nom);
+	}
+	FOPEN_OR_ERR_RET(f_cmax, ret);
+
+	s_nom = fgets(buf_nom, sizeof(buf_nom), f_nom);
+	FOPS_OR_NULL_GOTO(s_nom, out);
+
+	s_max = fgets(buf_max, sizeof(buf_max), f_max);
+	FOPS_OR_NULL_GOTO(s_max, out);
+
+	s_cmax = fgets(buf_cmax, sizeof(buf_cmax), f_cmax);
+	FOPS_OR_NULL_GOTO(s_cmax, out);
+
+	/* Strip the line break if there is */
+	p_nom = strchr(buf_nom, '\n');
+	if (p_nom != NULL)
+		*p_nom = 0;
+
+	p_max = strchr(buf_max, '\n');
+	if (p_max != NULL)
+		*p_max = 0;
+
+	p_cmax = strchr(buf_cmax, '\n');
+	if (p_cmax != NULL)
+		*p_cmax = 0;
+
+	nominal_perf = strtoul(buf_nom, &p_nom, POWER_CONVERT_TO_DECIMAL);
+	highest_perf = strtoul(buf_max, &p_max, POWER_CONVERT_TO_DECIMAL);
+	cpuinfo_max_freq = strtoul(buf_cmax, &p_cmax, POWER_CONVERT_TO_DECIMAL);
+
+	pi->highest_perf = highest_perf;
+	pi->nominal_perf = nominal_perf;
+
+	if ((highest_perf > nominal_perf) && ((cpuinfo_max_freq == highest_perf) ||
+				cpuinfo_max_freq == highest_perf * UNIT_DIFF)) {
+		pi->turbo_available = 1;
+		pi->turbo_enable = 1;
+		ret = 0;
+		POWER_DEBUG_TRACE("Lcore %u can do Turbo Boost! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	} else {
+		pi->turbo_available = 0;
+		pi->turbo_enable = 0;
+		POWER_DEBUG_TRACE("Lcore %u Turbo not available! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	}
+
+out:
+	fclose(f_nom);
+	fclose(f_max);
+	fclose(f_cmax);
+
+	return ret;
+}
+
+/**
+ * It is to get the available frequencies of the specific lcore by reading the
+ * sys file.
+ */
+static int
+power_get_available_freqs(struct cppc_power_info *pi)
+{
+	FILE *f_min, *f_max;
+	int ret = -1;
+	char *p_min, *p_max;
+	char buf_min[BUFSIZ];
+	char buf_max[BUFSIZ];
+	char fullpath_min[PATH_MAX];
+	char fullpath_max[PATH_MAX];
+	char *s_min, *s_max;
+	uint32_t scaling_min_freq = 0, scaling_max_freq = 0, nominal_perf = 0;
+	uint32_t i, num_freqs = 0;
+
+	snprintf(fullpath_max, sizeof(fullpath_max),
+			POWER_SYSFILE_SCALING_MAX_FREQ,
+			pi->lcore_id);
+	snprintf(fullpath_min, sizeof(fullpath_min),
+			POWER_SYSFILE_SCALING_MIN_FREQ,
+			pi->lcore_id);
+
+	f_min = fopen(fullpath_min, "r");
+	FOPEN_OR_ERR_RET(f_min, ret);
+
+	f_max = fopen(fullpath_max, "r");
+	if (f_max == NULL)
+		fclose(f_min);
+
+	FOPEN_OR_ERR_RET(f_max, ret);
+
+	s_min = fgets(buf_min, sizeof(buf_min), f_min);
+	FOPS_OR_NULL_GOTO(s_min, out);
+
+	s_max = fgets(buf_max, sizeof(buf_max), f_max);
+	FOPS_OR_NULL_GOTO(s_max, out);
+
+
+	/* Strip the line break if there is */
+	p_min = strchr(buf_min, '\n');
+	if (p_min != NULL)
+		*p_min = 0;
+
+	p_max = strchr(buf_max, '\n');
+	if (p_max != NULL)
+		*p_max = 0;
+
+	scaling_min_freq = strtoul(buf_min, &p_min, POWER_CONVERT_TO_DECIMAL);
+	scaling_max_freq = strtoul(buf_max, &p_max, POWER_CONVERT_TO_DECIMAL);
+
+	power_check_turbo(pi);
+
+	if (scaling_max_freq < scaling_min_freq)
+		goto out;
+
+	/* If turbo is available then there is one extra freq bucket
+	 * to store the sys max freq which value is scaling_max_freq
+	 */
+	nominal_perf = (pi->nominal_perf < UNIT_DIFF) ?
+			pi->nominal_perf * UNIT_DIFF : pi->nominal_perf;
+	num_freqs = (nominal_perf - scaling_min_freq) / BUS_FREQ + 1 +
+		pi->turbo_available;
+
+	/* Generate the freq bucket array. */
+	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
+		if ((i == 0) && pi->turbo_available)
+			pi->freqs[pi->nb_freqs++] = scaling_max_freq;
+		else
+			pi->freqs[pi->nb_freqs++] =
+			nominal_perf - (i - pi->turbo_available) * BUS_FREQ;
+	}
+
+	ret = 0;
+
+	POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
+			num_freqs, pi->lcore_id);
+
+out:
+	fclose(f_min);
+	fclose(f_max);
+
+	return ret;
+}
+
+/**
+ * It is to fopen the sys file for the future setting the lcore frequency.
+ */
+static int
+power_init_for_setting_freq(struct cppc_power_info *pi)
+{
+	FILE *f;
+	char fullpath[PATH_MAX];
+	char buf[BUFSIZ];
+	uint32_t i, freq;
+	char *s;
+
+	snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_SETSPEED,
+			pi->lcore_id);
+	f = fopen(fullpath, "rw+");
+	FOPEN_OR_ERR_RET(f, -1);
+
+	s = fgets(buf, sizeof(buf), f);
+	FOPS_OR_NULL_GOTO(s, out);
+
+	freq = strtoul(buf, NULL, POWER_CONVERT_TO_DECIMAL);
+
+	/* convert the frequency to nearest 100000 value
+	 * Ex: if freq=1396789 then freq_conv=1400000
+	 * Ex: if freq=800030 then freq_conv=800000
+	 */
+	unsigned int freq_conv = 0;
+	freq_conv = (freq + FREQ_ROUNDING_DELTA)
+				/ ROUND_FREQ_TO_N_100000;
+	freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
+
+	for (i = 0; i < pi->nb_freqs; i++) {
+		if (freq_conv == pi->freqs[i]) {
+			pi->curr_idx = i;
+			pi->f = f;
+			return 0;
+		}
+	}
+
+out:
+	fclose(f);
+
+	return -1;
+}
+
+int
+power_cppc_cpufreq_check_supported(void)
+{
+	return cpufreq_check_scaling_driver(POWER_CPPC_DRIVER);
+}
+
+int
+power_cppc_cpufreq_init(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_IDLE;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"in use\n", lcore_id);
+		return -1;
+	}
+
+	pi->lcore_id = lcore_id;
+	/* Check and set the governor */
+	if (power_set_governor_userspace(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
+				"userspace\n", lcore_id);
+		goto fail;
+	}
+
+	/* Get the available frequencies */
+	if (power_get_available_freqs(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Init for setting lcore frequency */
+	if (power_init_for_setting_freq(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Set freq to max by default */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
+				"to max\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
+			"power management\n", lcore_id);
+
+	__atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	return -1;
+}
+
+/**
+ * It is to check the governor and then set the original governor back if
+ * needed by writing the sys file.
+ */
+static int
+power_set_governor_original(struct cppc_power_info *pi)
+{
+	FILE *f;
+	int ret = -1;
+	char buf[BUFSIZ];
+	char fullpath[PATH_MAX];
+	char *s;
+	int val;
+
+	snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
+			pi->lcore_id);
+	f = fopen(fullpath, "rw+");
+	FOPEN_OR_ERR_RET(f, ret);
+
+	s = fgets(buf, sizeof(buf), f);
+	FOPS_OR_NULL_GOTO(s, out);
+
+	/* Check if the governor to be set is the same as current */
+	if (strncmp(buf, pi->governor_ori, sizeof(pi->governor_ori)) == 0) {
+		ret = 0;
+		POWER_DEBUG_TRACE("Power management governor of lcore %u "
+				"has already been set to %s\n",
+				pi->lcore_id, pi->governor_ori);
+		goto out;
+	}
+
+	/* Write back the original governor */
+	val = fseek(f, 0, SEEK_SET);
+	FOPS_OR_ERR_GOTO(val, out);
+
+	val = fputs(pi->governor_ori, f);
+	FOPS_OR_ERR_GOTO(val, out);
+
+	ret = 0;
+	RTE_LOG(INFO, POWER, "Power management governor of lcore %u "
+			"has been set back to %s successfully\n",
+			pi->lcore_id, pi->governor_ori);
+out:
+	fclose(f);
+
+	return ret;
+}
+
+int
+power_cppc_cpufreq_exit(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_USED;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"not used\n", lcore_id);
+		return -1;
+	}
+
+	/* Close FD of setting freq */
+	fclose(pi->f);
+	pi->f = NULL;
+
+	/* Set the governor back to the original */
+	if (power_set_governor_original(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
+				"to the original\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
+			"'userspace' mode and been set back to the "
+			"original\n", lcore_id);
+	__atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+
+	return -1;
+}
+
+uint32_t
+power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return 0;
+	}
+
+	if (freqs == NULL) {
+		RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
+		return 0;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (num < pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
+		return 0;
+	}
+	rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
+
+	return pi->nb_freqs;
+}
+
+uint32_t
+power_cppc_cpufreq_get_freq(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return RTE_POWER_INVALID_FREQ_INDEX;
+	}
+
+	return lcore_power_info[lcore_id].curr_idx;
+}
+
+int
+power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	return set_freq_internal(&(lcore_power_info[lcore_id]), index);
+}
+
+int
+power_cppc_cpufreq_freq_down(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx + 1 == pi->nb_freqs)
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx + 1);
+}
+
+int
+power_cppc_cpufreq_freq_up(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx == 0 || (pi->curr_idx == 1 &&
+		pi->turbo_available && !pi->turbo_enable))
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx - 1);
+}
+
+int
+power_cppc_cpufreq_freq_max(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	/* Frequencies in the array are from high to low. */
+	if (lcore_power_info[lcore_id].turbo_available) {
+		if (lcore_power_info[lcore_id].turbo_enable)
+			/* Set to Turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 0);
+		else
+			/* Set to max non-turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 1);
+	} else
+		return set_freq_internal(&lcore_power_info[lcore_id], 0);
+}
+
+int
+power_cppc_cpufreq_freq_min(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->nb_freqs - 1);
+}
+
+int
+power_cppc_turbo_status(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	return pi->turbo_enable;
+}
+
+int
+power_cppc_enable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	if (pi->turbo_available)
+		pi->turbo_enable = 1;
+	else {
+		pi->turbo_enable = 0;
+		RTE_LOG(ERR, POWER,
+			"Failed to enable turbo on lcore %u\n",
+			lcore_id);
+		return -1;
+	}
+
+	/* TODO: must set to max once enbling Turbo? Considering add condition:
+	 * if ((pi->turbo_available) && (pi->curr_idx <= 1))
+	 */
+	/* Max may have changed, so call to max function */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER,
+			"Failed to set frequency of lcore %u to max\n",
+			lcore_id);
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+power_cppc_disable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	pi->turbo_enable = 0;
+
+	if ((pi->turbo_available) && (pi->curr_idx <= 1)) {
+		/* Try to set freq to max by default coming out of turbo */
+		if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+			RTE_LOG(ERR, POWER,
+				"Failed to set frequency of lcore %u to max\n",
+				lcore_id);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+int
+power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+	if (caps == NULL) {
+		RTE_LOG(ERR, POWER, "Invalid argument\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	caps->capabilities = 0;
+	caps->turbo = !!(pi->turbo_available);
+
+	return 0;
+}
diff --git a/lib/power/power_cppc_cpufreq.h b/lib/power/power_cppc_cpufreq.h
new file mode 100644
index 000000000..427ce4f83
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.h
@@ -0,0 +1,229 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#ifndef _POWER_CPPC_CPUFREQ_H
+#define _POWER_CPPC_CPUFREQ_H
+
+/**
+ * @file
+ * RTE Power Management via userspace CPPC cpufreq
+ */
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_string_fns.h>
+#include "rte_power.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Check if CPPC power management is supported.
+ *
+ * @return
+ *   - 1 if supported
+ *   - 0 if unsupported
+ *   - -1 if error, with rte_errno indicating reason for error.
+ */
+int power_cppc_cpufreq_check_supported(void);
+
+/**
+ * Initialize power management for a specific lcore. It will check and set the
+ * governor to userspace for the lcore, get the available frequencies, and
+ * prepare to set new lcore frequency.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_init(unsigned int lcore_id);
+
+/**
+ * Exit power management on a specific lcore. It will set the governor to which
+ * is before initialized.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_exit(unsigned int lcore_id);
+
+/**
+ * Get the available frequencies of a specific lcore. The return value will be
+ * the minimal one of the total number of available frequencies and the number
+ * of buffer. The index of available frequencies used in other interfaces
+ * should be in the range of 0 to this return value.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param freqs
+ *  The buffer array to save the frequencies.
+ * @param num
+ *  The number of frequencies to get.
+ *
+ * @return
+ *  The number of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs,
+		uint32_t num);
+
+/**
+ * Return the current index of available frequencies of a specific lcore. It
+ * will return 'RTE_POWER_INVALID_FREQ_INDEX = (~0)' if error.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  The current index of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_get_freq(unsigned int lcore_id);
+
+/**
+ * Set the new frequency for a specific lcore by indicating the index of
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param index
+ *  The index of available frequencies.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index);
+
+/**
+ * Scale up the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_up(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_down(unsigned int lcore_id);
+
+/**
+ * Scale up the frequency of a specific lcore to the highest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_max(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore to the lowest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_min(unsigned int lcore_id);
+
+/**
+ * Get the turbo status of a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 Turbo Boost is enabled on this lcore.
+ *  - 0 Turbo Boost is disabled on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_turbo_status(unsigned int lcore_id);
+
+/**
+ * Enable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost is enabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_enable_turbo(unsigned int lcore_id);
+
+/**
+ * Disable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost disabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_disable_turbo(unsigned int lcore_id);
+
+/**
+ * Returns power capabilities for a specific lcore.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param caps
+ *  pointer to rte_power_core_capabilities object.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/power/rte_power.c b/lib/power/rte_power.c
index 98eaba915..3d51ff8be 100644
--- a/lib/power/rte_power.c
+++ b/lib/power/rte_power.c
@@ -10,6 +10,7 @@
 #include "power_kvm_vm.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
+#include "power_cppc_cpufreq.h"
 
 enum power_management_env global_default_env = PM_ENV_NOT_SET;
 
@@ -54,6 +55,8 @@ rte_power_check_env_supported(enum power_management_env env)
 		return power_pstate_cpufreq_check_supported();
 	case PM_ENV_KVM_VM:
 		return power_kvm_vm_check_supported();
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_check_supported();
 	default:
 		rte_errno = EINVAL;
 		return -1;
@@ -110,6 +113,18 @@ rte_power_set_env(enum power_management_env env)
 		rte_power_freq_disable_turbo = power_pstate_disable_turbo;
 		rte_power_get_capabilities = power_pstate_get_capabilities;
 
+	} else if (env == PM_ENV_CPPC_CPUFREQ) {
+		rte_power_freqs = power_cppc_cpufreq_freqs;
+		rte_power_get_freq = power_cppc_cpufreq_get_freq;
+		rte_power_set_freq = power_cppc_cpufreq_set_freq;
+		rte_power_freq_up = power_cppc_cpufreq_freq_up;
+		rte_power_freq_down = power_cppc_cpufreq_freq_down;
+		rte_power_freq_min = power_cppc_cpufreq_freq_min;
+		rte_power_freq_max = power_cppc_cpufreq_freq_max;
+		rte_power_turbo_status = power_cppc_turbo_status;
+		rte_power_freq_enable_turbo = power_cppc_enable_turbo;
+		rte_power_freq_disable_turbo = power_cppc_disable_turbo;
+		rte_power_get_capabilities = power_cppc_get_capabilities;
 	} else {
 		RTE_LOG(ERR, POWER, "Invalid Power Management Environment(%d) set\n",
 				env);
@@ -153,6 +168,8 @@ rte_power_init(unsigned int lcore_id)
 		return power_kvm_vm_init(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_init(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_init(lcore_id);
 	default:
 		RTE_LOG(INFO, POWER, "Env isn't set yet!\n");
 	}
@@ -172,6 +189,13 @@ rte_power_init(unsigned int lcore_id)
 		goto out;
 	}
 
+	RTE_LOG(INFO, POWER, "Attempting to initialise CPPC power management...\n");
+	ret = power_cppc_cpufreq_init(lcore_id);
+	if (ret == 0) {
+		rte_power_set_env(PM_ENV_CPPC_CPUFREQ);
+		goto out;
+	}
+
 	RTE_LOG(INFO, POWER, "Attempting to initialise VM power management...\n");
 	ret = power_kvm_vm_init(lcore_id);
 	if (ret == 0) {
@@ -194,6 +218,8 @@ rte_power_exit(unsigned int lcore_id)
 		return power_kvm_vm_exit(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_exit(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_exit(lcore_id);
 	default:
 		RTE_LOG(ERR, POWER, "Environment has not been set, unable to exit gracefully\n");
 
diff --git a/lib/power/rte_power.h b/lib/power/rte_power.h
index c8086bf6b..c5759afa3 100644
--- a/lib/power/rte_power.h
+++ b/lib/power/rte_power.h
@@ -22,7 +22,7 @@ extern "C" {
 
 /* Power Management Environment State */
 enum power_management_env {PM_ENV_NOT_SET, PM_ENV_ACPI_CPUFREQ, PM_ENV_KVM_VM,
-		PM_ENV_PSTATE_CPUFREQ};
+		PM_ENV_PSTATE_CPUFREQ, PM_ENV_CPPC_CPUFREQ};
 
 /**
  * @warning
-- 
2.20.1

Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc cpufreq

[Burakov, Anatoly]

On 22-Apr-21 7:15 AM, Richael Zhuang wrote:
> Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> supported, which are both not available on arm64 platforms. Add
> support for cppc_cpufreq driver which works on most arm64 platforms.
> 
> Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> ---

Just a general note: this looks like a copy-paste of pstate code. Which 
is perfectly fine, except that we can do better than copying some faults 
of the pstate code to other drivers. I've submitted a patch [1] 
attempting to fix some of the pressing issues and code duplication in 
pstate driver, but i'm sure with a fresh driver, you can do even better :)

[1] 
http://patches.dpdk.org/project/dpdk/patch/20210402092701.258316-1-anatoly.burakov@intel.com/

-- 
Thanks,
Anatoly

Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc cpufreq

[Richael Zhuang]

> -----Original Message-----
> From: Burakov, Anatoly <anatoly.burakov@intel.com>
> Sent: Thursday, April 22, 2021 5:06 PM
> To: Richael Zhuang <Richael.Zhuang@arm.com>; dev@dpdk.org
> Cc: nd <nd@arm.com>; David Hunt <david.hunt@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc cpufreq
> 
> On 22-Apr-21 7:15 AM, Richael Zhuang wrote:
> > Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> > supported, which are both not available on arm64 platforms. Add
> > support for cppc_cpufreq driver which works on most arm64 platforms.
> >
> > Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> > ---
> 
> Just a general note: this looks like a copy-paste of pstate code. Which is
> perfectly fine, except that we can do better than copying some faults of the
> pstate code to other drivers. I've submitted a patch [1] attempting to fix
> some of the pressing issues and code duplication in pstate driver, but i'm
> sure with a fresh driver, you can do even better :)
> 
> [1]
> http://patches.dpdk.org/project/dpdk/patch/20210402092701.258316-1-
> anatoly.burakov@intel.com/
> 
> --
> Thanks,
> Anatoly

For CPPC is defined in acpi v5.0+ spec,  I reused most code in acpi_cpufreq  to get a quick workable version on our platform with only cppc driver. I have verified  its basic functions. If you find some problems please help to point out thus I can rework it. Thanks .

Best Regards,
Richael

Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc cpufreq

[Burakov, Anatoly]

On 22-Apr-21 10:29 AM, Richael Zhuang wrote:
> 
> 
>> -----Original Message-----
>> From: Burakov, Anatoly <anatoly.burakov@intel.com>
>> Sent: Thursday, April 22, 2021 5:06 PM
>> To: Richael Zhuang <Richael.Zhuang@arm.com>; dev@dpdk.org
>> Cc: nd <nd@arm.com>; David Hunt <david.hunt@intel.com>
>> Subject: Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc cpufreq
>>
>> On 22-Apr-21 7:15 AM, Richael Zhuang wrote:
>>> Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
>>> supported, which are both not available on arm64 platforms. Add
>>> support for cppc_cpufreq driver which works on most arm64 platforms.
>>>
>>> Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
>>> ---
>>
>> Just a general note: this looks like a copy-paste of pstate code. Which is
>> perfectly fine, except that we can do better than copying some faults of the
>> pstate code to other drivers. I've submitted a patch [1] attempting to fix
>> some of the pressing issues and code duplication in pstate driver, but i'm
>> sure with a fresh driver, you can do even better :)
>>
>> [1]
>> http://patches.dpdk.org/project/dpdk/patch/20210402092701.258316-1-
>> anatoly.burakov@intel.com/
>>
>> --
>> Thanks,
>> Anatoly
> 
> For CPPC is defined in acpi v5.0+ spec,  I reused most code in acpi_cpufreq  to get a quick workable version on our platform with only cppc driver. I have verified  its basic functions. If you find some problems please help to point out thus I can rework it. Thanks .
> 
> Best Regards,
> Richael
> 

Well, pstate code was copied from ACPI so it does share the same flaws:

- Lots of code duplication (e.g. snprintf for filename, fopen sequences, 
etc.)
- Confusing and bug-prone error handling (e.g. return macros in the 
middle of a function)
- Mixing power management logic and gory details of string handling

Good examples of the above are in your `power_check_turbo()` function - 
lots of string handling code interspersed with file opens, and actual 
logic of power management.

Please see the patch i linked earlier [1] to understand what kind of 
changes i'm suggesting. Perhaps you could do even better :)

[1] 
http://patches.dpdk.org/project/dpdk/patch/20210402092701.258316-1-anatoly.burakov@intel.com/

-- 
Thanks,
Anatoly

Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc cpufreq

[Richael Zhuang]

> -----Original Message-----
> From: Burakov, Anatoly <anatoly.burakov@intel.com>
> Sent: Thursday, April 22, 2021 6:00 PM
> To: Richael Zhuang <Richael.Zhuang@arm.com>; dev@dpdk.org
> Cc: nd <nd@arm.com>; David Hunt <david.hunt@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc cpufreq
> 
> On 22-Apr-21 10:29 AM, Richael Zhuang wrote:
> >
> >
> >> -----Original Message-----
> >> From: Burakov, Anatoly <anatoly.burakov@intel.com>
> >> Sent: Thursday, April 22, 2021 5:06 PM
> >> To: Richael Zhuang <Richael.Zhuang@arm.com>; dev@dpdk.org
> >> Cc: nd <nd@arm.com>; David Hunt <david.hunt@intel.com>
> >> Subject: Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc
> >> cpufreq
> >>
> >> On 22-Apr-21 7:15 AM, Richael Zhuang wrote:
> >>> Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> >>> supported, which are both not available on arm64 platforms. Add
> >>> support for cppc_cpufreq driver which works on most arm64 platforms.
> >>>
> >>> Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> >>> ---
> >>
> >> Just a general note: this looks like a copy-paste of pstate code.
> >> Which is perfectly fine, except that we can do better than copying
> >> some faults of the pstate code to other drivers. I've submitted a
> >> patch [1] attempting to fix some of the pressing issues and code
> >> duplication in pstate driver, but i'm sure with a fresh driver, you
> >> can do even better :)
> >>
> >> [1]
> >> http://patches.dpdk.org/project/dpdk/patch/20210402092701.258316-1-
> >> anatoly.burakov@intel.com/
> >>
> >> --
> >> Thanks,
> >> Anatoly
> >
> > For CPPC is defined in acpi v5.0+ spec,  I reused most code in acpi_cpufreq
> to get a quick workable version on our platform with only cppc driver. I have
> verified  its basic functions. If you find some problems please help to point
> out thus I can rework it. Thanks .
> >
> > Best Regards,
> > Richael
> >
> 
> Well, pstate code was copied from ACPI so it does share the same flaws:
> 
> - Lots of code duplication (e.g. snprintf for filename, fopen sequences,
> etc.)
> - Confusing and bug-prone error handling (e.g. return macros in the middle
> of a function)
> - Mixing power management logic and gory details of string handling
> 
> Good examples of the above are in your `power_check_turbo()` function -
> lots of string handling code interspersed with file opens, and actual logic of
> power management.
> 
> Please see the patch i linked earlier [1] to understand what kind of changes
> i'm suggesting. Perhaps you could do even better :)
> 
> [1]
> http://patches.dpdk.org/project/dpdk/patch/20210402092701.258316-1-
> anatoly.burakov@intel.com/
> 
> --
> Thanks,
> Anatoly
Thanks. I'll rework it to make it look better.

Best Regards,
Richael

Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc cpufreq

[Burakov, Anatoly]

On 22-Apr-21 11:02 AM, Richael Zhuang wrote:
> 
> 
>> -----Original Message-----
>> From: Burakov, Anatoly <anatoly.burakov@intel.com>
>> Sent: Thursday, April 22, 2021 6:00 PM
>> To: Richael Zhuang <Richael.Zhuang@arm.com>; dev@dpdk.org
>> Cc: nd <nd@arm.com>; David Hunt <david.hunt@intel.com>
>> Subject: Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc cpufreq
>>
>> On 22-Apr-21 10:29 AM, Richael Zhuang wrote:
>>>
>>>
>>>> -----Original Message-----
>>>> From: Burakov, Anatoly <anatoly.burakov@intel.com>
>>>> Sent: Thursday, April 22, 2021 5:06 PM
>>>> To: Richael Zhuang <Richael.Zhuang@arm.com>; dev@dpdk.org
>>>> Cc: nd <nd@arm.com>; David Hunt <david.hunt@intel.com>
>>>> Subject: Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc
>>>> cpufreq
>>>>
>>>> On 22-Apr-21 7:15 AM, Richael Zhuang wrote:
>>>>> Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
>>>>> supported, which are both not available on arm64 platforms. Add
>>>>> support for cppc_cpufreq driver which works on most arm64 platforms.
>>>>>
>>>>> Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
>>>>> ---
>>>>
>>>> Just a general note: this looks like a copy-paste of pstate code.
>>>> Which is perfectly fine, except that we can do better than copying
>>>> some faults of the pstate code to other drivers. I've submitted a
>>>> patch [1] attempting to fix some of the pressing issues and code
>>>> duplication in pstate driver, but i'm sure with a fresh driver, you
>>>> can do even better :)
>>>>
>>>> [1]
>>>> http://patches.dpdk.org/project/dpdk/patch/20210402092701.258316-1-
>>>> anatoly.burakov@intel.com/
>>>>
>>>> --
>>>> Thanks,
>>>> Anatoly
>>>
>>> For CPPC is defined in acpi v5.0+ spec,  I reused most code in acpi_cpufreq
>> to get a quick workable version on our platform with only cppc driver. I have
>> verified  its basic functions. If you find some problems please help to point
>> out thus I can rework it. Thanks .
>>>
>>> Best Regards,
>>> Richael
>>>
>>
>> Well, pstate code was copied from ACPI so it does share the same flaws:
>>
>> - Lots of code duplication (e.g. snprintf for filename, fopen sequences,
>> etc.)
>> - Confusing and bug-prone error handling (e.g. return macros in the middle
>> of a function)
>> - Mixing power management logic and gory details of string handling
>>
>> Good examples of the above are in your `power_check_turbo()` function -
>> lots of string handling code interspersed with file opens, and actual logic of
>> power management.
>>
>> Please see the patch i linked earlier [1] to understand what kind of changes
>> i'm suggesting. Perhaps you could do even better :)
>>
>> [1]
>> http://patches.dpdk.org/project/dpdk/patch/20210402092701.258316-1-
>> anatoly.burakov@intel.com/
>>
>> --
>> Thanks,
>> Anatoly
> Thanks. I'll rework it to make it look better.
> 
> Best Regards,
> Richael
> 

Hi,

FYI i've updated my refactor patch [1] so that you could base your work 
off it and not have to do most of it yourself. Feel free to take over 
the patchset if you like!

[1] 
http://patches.dpdk.org/project/dpdk/patch/83b1e89d14834251d4d7e72fcc19d82dfb52686d.1619175790.git.anatoly.burakov@intel.com/
-- 
Thanks,
Anatoly

Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc cpufreq

[Richael Zhuang]

> -----Original Message-----
> From: Burakov, Anatoly <anatoly.burakov@intel.com>
> Sent: Friday, April 23, 2021 7:38 PM
> To: Richael Zhuang <Richael.Zhuang@arm.com>; dev@dpdk.org
> Cc: nd <nd@arm.com>; David Hunt <david.hunt@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc cpufreq
> 
> On 22-Apr-21 11:02 AM, Richael Zhuang wrote:
> >
> >
> >> -----Original Message-----
> >> From: Burakov, Anatoly <anatoly.burakov@intel.com>
> >> Sent: Thursday, April 22, 2021 6:00 PM
> >> To: Richael Zhuang <Richael.Zhuang@arm.com>; dev@dpdk.org
> >> Cc: nd <nd@arm.com>; David Hunt <david.hunt@intel.com>
> >> Subject: Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc
> >> cpufreq
> >>
> >> On 22-Apr-21 10:29 AM, Richael Zhuang wrote:
> >>>
> >>>
> >>>> -----Original Message-----
> >>>> From: Burakov, Anatoly <anatoly.burakov@intel.com>
> >>>> Sent: Thursday, April 22, 2021 5:06 PM
> >>>> To: Richael Zhuang <Richael.Zhuang@arm.com>; dev@dpdk.org
> >>>> Cc: nd <nd@arm.com>; David Hunt <david.hunt@intel.com>
> >>>> Subject: Re: [dpdk-dev] [PATCH v1 1/1] power: add support for cppc
> >>>> cpufreq
> >>>>
> >>>> On 22-Apr-21 7:15 AM, Richael Zhuang wrote:
> >>>>> Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> >>>>> supported, which are both not available on arm64 platforms. Add
> >>>>> support for cppc_cpufreq driver which works on most arm64
> platforms.
> >>>>>
> >>>>> Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> >>>>> ---
> >>>>
> >>>> Just a general note: this looks like a copy-paste of pstate code.
> >>>> Which is perfectly fine, except that we can do better than copying
> >>>> some faults of the pstate code to other drivers. I've submitted a
> >>>> patch [1] attempting to fix some of the pressing issues and code
> >>>> duplication in pstate driver, but i'm sure with a fresh driver, you
> >>>> can do even better :)
> >>>>
> >>>> [1]
> >>>> http://patches.dpdk.org/project/dpdk/patch/20210402092701.258316-
> 1-
> >>>> anatoly.burakov@intel.com/
> >>>>
> >>>> --
> >>>> Thanks,
> >>>> Anatoly
> >>>
> >>> For CPPC is defined in acpi v5.0+ spec,  I reused most code in
> >>> acpi_cpufreq
> >> to get a quick workable version on our platform with only cppc
> >> driver. I have verified  its basic functions. If you find some
> >> problems please help to point out thus I can rework it. Thanks .
> >>>
> >>> Best Regards,
> >>> Richael
> >>>
> >>
> >> Well, pstate code was copied from ACPI so it does share the same flaws:
> >>
> >> - Lots of code duplication (e.g. snprintf for filename, fopen
> >> sequences,
> >> etc.)
> >> - Confusing and bug-prone error handling (e.g. return macros in the
> >> middle of a function)
> >> - Mixing power management logic and gory details of string handling
> >>
> >> Good examples of the above are in your `power_check_turbo()` function
> >> - lots of string handling code interspersed with file opens, and
> >> actual logic of power management.
> >>
> >> Please see the patch i linked earlier [1] to understand what kind of
> >> changes i'm suggesting. Perhaps you could do even better :)
> >>
> >> [1]
> >> http://patches.dpdk.org/project/dpdk/patch/20210402092701.258316-1-
> >> anatoly.burakov@intel.com/
> >>
> >> --
> >> Thanks,
> >> Anatoly
> > Thanks. I'll rework it to make it look better.
> >
> > Best Regards,
> > Richael
> >
> 
> Hi,
> 
> FYI i've updated my refactor patch [1] so that you could base your work off it
> and not have to do most of it yourself. Feel free to take over the patchset if
> you like!
> 
> [1]
> http://patches.dpdk.org/project/dpdk/patch/83b1e89d14834251d4d7e72fcc
> 19d82dfb52686d.1619175790.git.anatoly.burakov@intel.com/
> --
> Thanks,
> Anatoly

Thanks. I will rework it and reduce some duplicated code in patch v2.

Best Regards,
Richael

[dpdk-dev] [PATCH v2 0/2] power: add support for cppc cpufreq driver

[Richael Zhuang]

v2:
rebase on Anatoly's patch: http://dpdk.org/patch/92077;
fix check_cur_freq bug: round cur freq only when using CPPC driver

Richael Zhuang (2):
  power: add support for cppc cpufreq
  test/power: round cpuinfo cur freq only when using CPPC cpufreq

 app/test/test_power.c          |   3 +-
 app/test/test_power_cpufreq.c  |  26 +-
 lib/power/meson.build          |   1 +
 lib/power/power_cppc_cpufreq.c | 632 +++++++++++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h | 229 ++++++++++++
 lib/power/rte_power.c          |  26 ++
 lib/power/rte_power.h          |   2 +-
 7 files changed, 907 insertions(+), 12 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

-- 
2.20.1

[dpdk-dev] [PATCH v2 1/2] power: add support for cppc cpufreq

[Richael Zhuang]

Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
supported, which are both not available on arm64 platforms. Add
support for cppc_cpufreq driver which works on most arm64 platforms.

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
---
 app/test/test_power.c          |   3 +-
 app/test/test_power_cpufreq.c  |   3 +-
 lib/power/meson.build          |   1 +
 lib/power/power_cppc_cpufreq.c | 632 +++++++++++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h | 229 ++++++++++++
 lib/power/rte_power.c          |  26 ++
 lib/power/rte_power.h          |   2 +-
 7 files changed, 893 insertions(+), 3 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

diff --git a/app/test/test_power.c b/app/test/test_power.c
index da1d67c0a..b7b556134 100644
--- a/app/test/test_power.c
+++ b/app/test/test_power.c
@@ -133,7 +133,8 @@ test_power(void)
 	/* Perform tests for valid environments.*/
 	const enum power_management_env envs[] = {PM_ENV_ACPI_CPUFREQ,
 			PM_ENV_KVM_VM,
-			PM_ENV_PSTATE_CPUFREQ};
+			PM_ENV_PSTATE_CPUFREQ,
+			PM_ENV_CPPC_CPUFREQ};
 
 	unsigned int i;
 	for (i = 0; i < RTE_DIM(envs); ++i) {
diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index f753d24ac..87d07191d 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -470,7 +470,8 @@ test_power_cpufreq(void)
 
 	/* Test environment configuration */
 	env = rte_power_get_env();
-	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ)) {
+	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ) &&
+			(env != PM_ENV_CPPC_CPUFREQ)) {
 		printf("Unexpectedly got an environment other than ACPI/PSTATE\n");
 		goto fail_all;
 	}
diff --git a/lib/power/meson.build b/lib/power/meson.build
index 74c5f3a29..4a5b07292 100644
--- a/lib/power/meson.build
+++ b/lib/power/meson.build
@@ -21,6 +21,7 @@ sources = files(
         'rte_power.c',
         'rte_power_empty_poll.c',
         'rte_power_pmd_mgmt.c',
+	'power_cppc_cpufreq.c',
 )
 headers = files(
         'rte_power.h',
diff --git a/lib/power/power_cppc_cpufreq.c b/lib/power/power_cppc_cpufreq.c
new file mode 100644
index 000000000..c4a71a3d3
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.c
@@ -0,0 +1,632 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#include <rte_memcpy.h>
+#include <rte_memory.h>
+
+#include "power_cppc_cpufreq.h"
+#include "power_common.h"
+
+/* macros used for rounding frequency to nearest 100000 */
+#define FREQ_ROUNDING_DELTA 50000
+#define ROUND_FREQ_TO_N_100000 100000
+
+/* the unit of highest_perf and nominal_perf differs on different arm platforms.
+ * For highest_perf, it maybe 300 or 3000000, both means 3.0GHz.
+ */
+#define UNIT_DIFF 10000
+
+#define POWER_CONVERT_TO_DECIMAL 10
+
+#define POWER_GOVERNOR_USERSPACE "userspace"
+#define POWER_SYSFILE_SETSPEED   \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_setspeed"
+#define POWER_SYSFILE_SCALING_MAX_FREQ \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
+#define POWER_SYSFILE_SCALING_MIN_FREQ  \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
+#define POWER_SYSFILE_HIGHEST_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/highest_perf"
+#define POWER_SYSFILE_NOMINAL_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/nominal_perf"
+#define POWER_SYSFILE_SYS_MAX \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
+
+#define POWER_CPPC_DRIVER "cppc-cpufreq"
+#define BUS_FREQ     100000
+
+enum power_state {
+	POWER_IDLE = 0,
+	POWER_ONGOING,
+	POWER_USED,
+	POWER_UNKNOWN
+};
+
+/**
+ * Power info per lcore.
+ */
+struct cppc_power_info {
+	unsigned int lcore_id;                   /**< Logical core id */
+	uint32_t state;                      /**< Power in use state */
+	FILE *f;                             /**< FD of scaling_setspeed */
+	char governor_ori[32];               /**< Original governor name */
+	uint32_t curr_idx;                   /**< Freq index in freqs array */
+	uint32_t highest_perf;		     /**< system wide max freq */
+	uint32_t nominal_perf;		     /**< system wide nominal freq */
+	uint16_t turbo_available;            /**< Turbo Boost available */
+	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
+	uint32_t nb_freqs;                   /**< number of available freqs */
+	uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
+} __rte_cache_aligned;
+
+static struct cppc_power_info lcore_power_info[RTE_MAX_LCORE];
+
+/**
+ * It is to set specific freq for specific logical core, according to the index
+ * of supported frequencies.
+ */
+static int
+set_freq_internal(struct cppc_power_info *pi, uint32_t idx)
+{
+	if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
+				"should be less than %u\n", idx, pi->nb_freqs);
+		return -1;
+	}
+
+	/* Check if it is the same as current */
+	if (idx == pi->curr_idx)
+		return 0;
+
+	POWER_DEBUG_TRACE("Frequency[%u] %u to be set for lcore %u\n",
+			idx, pi->freqs[idx], pi->lcore_id);
+	if (fseek(pi->f, 0, SEEK_SET) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
+				"for setting frequency for lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	if (fprintf(pi->f, "%u", pi->freqs[idx]) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new frequency for "
+				"lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	fflush(pi->f);
+	pi->curr_idx = idx;
+
+	return 1;
+}
+
+/**
+ * It is to check the current scaling governor by reading sys file, and then
+ * set it into 'userspace' if it is not by writing the sys file. The original
+ * governor will be saved for rolling back.
+ */
+static int
+power_set_governor_userspace(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, POWER_GOVERNOR_USERSPACE,
+			pi->governor_ori, sizeof(pi->governor_ori));
+}
+
+static int
+power_check_turbo(struct cppc_power_info *pi)
+{
+	FILE *f_nom, *f_max, *f_cmax;
+	int ret = -1;
+	uint32_t nominal_perf = 0, highest_perf = 0, cpuinfo_max_freq = 0;
+
+	open_core_sysfs_file(POWER_SYSFILE_HIGHEST_PERF, pi->lcore_id, "r",
+			&f_max);
+	FOPEN_OR_ERR_GOTO(f_max, err);
+
+	open_core_sysfs_file(POWER_SYSFILE_NOMINAL_PERF, pi->lcore_id, "r",
+			&f_nom);
+	FOPEN_OR_ERR_GOTO(f_nom, err);
+
+	open_core_sysfs_file(POWER_SYSFILE_SYS_MAX, pi->lcore_id, "r",
+			&f_cmax);
+	FOPEN_OR_ERR_GOTO(f_cmax, err);
+
+	ret = read_core_sysfs_u32(f_max, &highest_perf);
+	FOPS_OR_ERR_GOTO(ret, err);
+
+	ret = read_core_sysfs_u32(f_nom, &nominal_perf);
+	FOPS_OR_ERR_GOTO(ret, err);
+
+	ret = read_core_sysfs_u32(f_cmax, &cpuinfo_max_freq);
+	FOPS_OR_ERR_GOTO(ret, err);
+
+	pi->highest_perf = highest_perf;
+	pi->nominal_perf = nominal_perf;
+
+	if ((highest_perf > nominal_perf) && ((cpuinfo_max_freq == highest_perf) ||
+				cpuinfo_max_freq == highest_perf * UNIT_DIFF)) {
+		pi->turbo_available = 1;
+		pi->turbo_enable = 1;
+		ret = 0;
+		POWER_DEBUG_TRACE("Lcore %u can do Turbo Boost! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	} else {
+		pi->turbo_available = 0;
+		pi->turbo_enable = 0;
+		POWER_DEBUG_TRACE("Lcore %u Turbo not available! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	}
+
+err:
+	if (f_max != NULL)
+		fclose(f_max);
+	if (f_nom != NULL)
+		fclose(f_nom);
+	if (f_cmax != NULL)
+		fclose(f_cmax);
+
+	return ret;
+}
+
+/**
+ * It is to get the available frequencies of the specific lcore by reading the
+ * sys file.
+ */
+static int
+power_get_available_freqs(struct cppc_power_info *pi)
+{
+	FILE *f_min, *f_max;
+	int ret = -1;
+	uint32_t scaling_min_freq = 0, scaling_max_freq = 0, nominal_perf = 0;
+	uint32_t i, num_freqs = 0;
+
+	open_core_sysfs_file(POWER_SYSFILE_SCALING_MAX_FREQ, pi->lcore_id, "r",
+			&f_max);
+	FOPEN_OR_ERR_GOTO(f_max, out);
+
+	open_core_sysfs_file(POWER_SYSFILE_SCALING_MIN_FREQ, pi->lcore_id, "r",
+			&f_min);
+	FOPEN_OR_ERR_GOTO(f_min, out);
+
+	ret = read_core_sysfs_u32(f_max, &scaling_max_freq);
+	FOPS_OR_ERR_GOTO(ret, out);
+
+	ret = read_core_sysfs_u32(f_min, &scaling_min_freq);
+	FOPS_OR_ERR_GOTO(ret, out);
+
+	power_check_turbo(pi);
+
+	if (scaling_max_freq < scaling_min_freq)
+		goto out;
+
+	/* If turbo is available then there is one extra freq bucket
+	 * to store the sys max freq which value is scaling_max_freq
+	 */
+	nominal_perf = (pi->nominal_perf < UNIT_DIFF) ?
+			pi->nominal_perf * UNIT_DIFF : pi->nominal_perf;
+	num_freqs = (nominal_perf - scaling_min_freq) / BUS_FREQ + 1 +
+		pi->turbo_available;
+
+	/* Generate the freq bucket array. */
+	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
+		if ((i == 0) && pi->turbo_available)
+			pi->freqs[pi->nb_freqs++] = scaling_max_freq;
+		else
+			pi->freqs[pi->nb_freqs++] =
+			nominal_perf - (i - pi->turbo_available) * BUS_FREQ;
+	}
+
+	ret = 0;
+
+	POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
+			num_freqs, pi->lcore_id);
+
+out:
+	if (f_min != NULL)
+		fclose(f_min);
+	if (f_max != NULL)
+		fclose(f_max);
+
+	return ret;
+}
+
+/**
+ * It is to fopen the sys file for the future setting the lcore frequency.
+ */
+static int
+power_init_for_setting_freq(struct cppc_power_info *pi)
+{
+	FILE *f;
+	char buf[BUFSIZ];
+	uint32_t i, freq;
+	int ret;
+
+	open_core_sysfs_file(POWER_SYSFILE_SETSPEED, pi->lcore_id, "rw+",
+			&f);
+	FOPEN_OR_ERR_GOTO(f, err);
+
+	ret = read_core_sysfs_s(f, buf, sizeof(buf));
+	FOPS_OR_ERR_GOTO(ret, err);
+
+	freq = strtoul(buf, NULL, POWER_CONVERT_TO_DECIMAL);
+
+	/* convert the frequency to nearest 100000 value
+	 * Ex: if freq=1396789 then freq_conv=1400000
+	 * Ex: if freq=800030 then freq_conv=800000
+	 */
+	unsigned int freq_conv = 0;
+	freq_conv = (freq + FREQ_ROUNDING_DELTA)
+				/ ROUND_FREQ_TO_N_100000;
+	freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
+
+	for (i = 0; i < pi->nb_freqs; i++) {
+		if (freq_conv == pi->freqs[i]) {
+			pi->curr_idx = i;
+			pi->f = f;
+			return 0;
+		}
+	}
+
+err:
+	if (f != NULL)
+		fclose(f);
+
+	return -1;
+}
+
+int
+power_cppc_cpufreq_check_supported(void)
+{
+	return cpufreq_check_scaling_driver(POWER_CPPC_DRIVER);
+}
+
+int
+power_cppc_cpufreq_init(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_IDLE;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"in use\n", lcore_id);
+		return -1;
+	}
+
+	pi->lcore_id = lcore_id;
+	/* Check and set the governor */
+	if (power_set_governor_userspace(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
+				"userspace\n", lcore_id);
+		goto fail;
+	}
+
+	/* Get the available frequencies */
+	if (power_get_available_freqs(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Init for setting lcore frequency */
+	if (power_init_for_setting_freq(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Set freq to max by default */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
+				"to max\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
+			"power management\n", lcore_id);
+
+	__atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	return -1;
+}
+
+/**
+ * It is to check the governor and then set the original governor back if
+ * needed by writing the sys file.
+ */
+static int
+power_set_governor_original(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, pi->governor_ori, NULL, 0);
+}
+
+int
+power_cppc_cpufreq_exit(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_USED;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"not used\n", lcore_id);
+		return -1;
+	}
+
+	/* Close FD of setting freq */
+	fclose(pi->f);
+	pi->f = NULL;
+
+	/* Set the governor back to the original */
+	if (power_set_governor_original(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
+				"to the original\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
+			"'userspace' mode and been set back to the "
+			"original\n", lcore_id);
+	__atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+
+	return -1;
+}
+
+uint32_t
+power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return 0;
+	}
+
+	if (freqs == NULL) {
+		RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
+		return 0;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (num < pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
+		return 0;
+	}
+	rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
+
+	return pi->nb_freqs;
+}
+
+uint32_t
+power_cppc_cpufreq_get_freq(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return RTE_POWER_INVALID_FREQ_INDEX;
+	}
+
+	return lcore_power_info[lcore_id].curr_idx;
+}
+
+int
+power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	return set_freq_internal(&(lcore_power_info[lcore_id]), index);
+}
+
+int
+power_cppc_cpufreq_freq_down(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx + 1 == pi->nb_freqs)
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx + 1);
+}
+
+int
+power_cppc_cpufreq_freq_up(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx == 0 || (pi->curr_idx == 1 &&
+		pi->turbo_available && !pi->turbo_enable))
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx - 1);
+}
+
+int
+power_cppc_cpufreq_freq_max(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	/* Frequencies in the array are from high to low. */
+	if (lcore_power_info[lcore_id].turbo_available) {
+		if (lcore_power_info[lcore_id].turbo_enable)
+			/* Set to Turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 0);
+		else
+			/* Set to max non-turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 1);
+	} else
+		return set_freq_internal(&lcore_power_info[lcore_id], 0);
+}
+
+int
+power_cppc_cpufreq_freq_min(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->nb_freqs - 1);
+}
+
+int
+power_cppc_turbo_status(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	return pi->turbo_enable;
+}
+
+int
+power_cppc_enable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	if (pi->turbo_available)
+		pi->turbo_enable = 1;
+	else {
+		pi->turbo_enable = 0;
+		RTE_LOG(ERR, POWER,
+			"Failed to enable turbo on lcore %u\n",
+			lcore_id);
+		return -1;
+	}
+
+	/* TODO: must set to max once enbling Turbo? Considering add condition:
+	 * if ((pi->turbo_available) && (pi->curr_idx <= 1))
+	 */
+	/* Max may have changed, so call to max function */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER,
+			"Failed to set frequency of lcore %u to max\n",
+			lcore_id);
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+power_cppc_disable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	pi->turbo_enable = 0;
+
+	if ((pi->turbo_available) && (pi->curr_idx <= 1)) {
+		/* Try to set freq to max by default coming out of turbo */
+		if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+			RTE_LOG(ERR, POWER,
+				"Failed to set frequency of lcore %u to max\n",
+				lcore_id);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+int
+power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+	if (caps == NULL) {
+		RTE_LOG(ERR, POWER, "Invalid argument\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	caps->capabilities = 0;
+	caps->turbo = !!(pi->turbo_available);
+
+	return 0;
+}
diff --git a/lib/power/power_cppc_cpufreq.h b/lib/power/power_cppc_cpufreq.h
new file mode 100644
index 000000000..427ce4f83
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.h
@@ -0,0 +1,229 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#ifndef _POWER_CPPC_CPUFREQ_H
+#define _POWER_CPPC_CPUFREQ_H
+
+/**
+ * @file
+ * RTE Power Management via userspace CPPC cpufreq
+ */
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_string_fns.h>
+#include "rte_power.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Check if CPPC power management is supported.
+ *
+ * @return
+ *   - 1 if supported
+ *   - 0 if unsupported
+ *   - -1 if error, with rte_errno indicating reason for error.
+ */
+int power_cppc_cpufreq_check_supported(void);
+
+/**
+ * Initialize power management for a specific lcore. It will check and set the
+ * governor to userspace for the lcore, get the available frequencies, and
+ * prepare to set new lcore frequency.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_init(unsigned int lcore_id);
+
+/**
+ * Exit power management on a specific lcore. It will set the governor to which
+ * is before initialized.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_exit(unsigned int lcore_id);
+
+/**
+ * Get the available frequencies of a specific lcore. The return value will be
+ * the minimal one of the total number of available frequencies and the number
+ * of buffer. The index of available frequencies used in other interfaces
+ * should be in the range of 0 to this return value.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param freqs
+ *  The buffer array to save the frequencies.
+ * @param num
+ *  The number of frequencies to get.
+ *
+ * @return
+ *  The number of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs,
+		uint32_t num);
+
+/**
+ * Return the current index of available frequencies of a specific lcore. It
+ * will return 'RTE_POWER_INVALID_FREQ_INDEX = (~0)' if error.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  The current index of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_get_freq(unsigned int lcore_id);
+
+/**
+ * Set the new frequency for a specific lcore by indicating the index of
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param index
+ *  The index of available frequencies.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index);
+
+/**
+ * Scale up the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_up(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_down(unsigned int lcore_id);
+
+/**
+ * Scale up the frequency of a specific lcore to the highest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_max(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore to the lowest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_min(unsigned int lcore_id);
+
+/**
+ * Get the turbo status of a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 Turbo Boost is enabled on this lcore.
+ *  - 0 Turbo Boost is disabled on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_turbo_status(unsigned int lcore_id);
+
+/**
+ * Enable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost is enabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_enable_turbo(unsigned int lcore_id);
+
+/**
+ * Disable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost disabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_disable_turbo(unsigned int lcore_id);
+
+/**
+ * Returns power capabilities for a specific lcore.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param caps
+ *  pointer to rte_power_core_capabilities object.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/power/rte_power.c b/lib/power/rte_power.c
index 98eaba915..3d51ff8be 100644
--- a/lib/power/rte_power.c
+++ b/lib/power/rte_power.c
@@ -10,6 +10,7 @@
 #include "power_kvm_vm.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
+#include "power_cppc_cpufreq.h"
 
 enum power_management_env global_default_env = PM_ENV_NOT_SET;
 
@@ -54,6 +55,8 @@ rte_power_check_env_supported(enum power_management_env env)
 		return power_pstate_cpufreq_check_supported();
 	case PM_ENV_KVM_VM:
 		return power_kvm_vm_check_supported();
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_check_supported();
 	default:
 		rte_errno = EINVAL;
 		return -1;
@@ -110,6 +113,18 @@ rte_power_set_env(enum power_management_env env)
 		rte_power_freq_disable_turbo = power_pstate_disable_turbo;
 		rte_power_get_capabilities = power_pstate_get_capabilities;
 
+	} else if (env == PM_ENV_CPPC_CPUFREQ) {
+		rte_power_freqs = power_cppc_cpufreq_freqs;
+		rte_power_get_freq = power_cppc_cpufreq_get_freq;
+		rte_power_set_freq = power_cppc_cpufreq_set_freq;
+		rte_power_freq_up = power_cppc_cpufreq_freq_up;
+		rte_power_freq_down = power_cppc_cpufreq_freq_down;
+		rte_power_freq_min = power_cppc_cpufreq_freq_min;
+		rte_power_freq_max = power_cppc_cpufreq_freq_max;
+		rte_power_turbo_status = power_cppc_turbo_status;
+		rte_power_freq_enable_turbo = power_cppc_enable_turbo;
+		rte_power_freq_disable_turbo = power_cppc_disable_turbo;
+		rte_power_get_capabilities = power_cppc_get_capabilities;
 	} else {
 		RTE_LOG(ERR, POWER, "Invalid Power Management Environment(%d) set\n",
 				env);
@@ -153,6 +168,8 @@ rte_power_init(unsigned int lcore_id)
 		return power_kvm_vm_init(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_init(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_init(lcore_id);
 	default:
 		RTE_LOG(INFO, POWER, "Env isn't set yet!\n");
 	}
@@ -172,6 +189,13 @@ rte_power_init(unsigned int lcore_id)
 		goto out;
 	}
 
+	RTE_LOG(INFO, POWER, "Attempting to initialise CPPC power management...\n");
+	ret = power_cppc_cpufreq_init(lcore_id);
+	if (ret == 0) {
+		rte_power_set_env(PM_ENV_CPPC_CPUFREQ);
+		goto out;
+	}
+
 	RTE_LOG(INFO, POWER, "Attempting to initialise VM power management...\n");
 	ret = power_kvm_vm_init(lcore_id);
 	if (ret == 0) {
@@ -194,6 +218,8 @@ rte_power_exit(unsigned int lcore_id)
 		return power_kvm_vm_exit(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_exit(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_exit(lcore_id);
 	default:
 		RTE_LOG(ERR, POWER, "Environment has not been set, unable to exit gracefully\n");
 
diff --git a/lib/power/rte_power.h b/lib/power/rte_power.h
index c8086bf6b..c5759afa3 100644
--- a/lib/power/rte_power.h
+++ b/lib/power/rte_power.h
@@ -22,7 +22,7 @@ extern "C" {
 
 /* Power Management Environment State */
 enum power_management_env {PM_ENV_NOT_SET, PM_ENV_ACPI_CPUFREQ, PM_ENV_KVM_VM,
-		PM_ENV_PSTATE_CPUFREQ};
+		PM_ENV_PSTATE_CPUFREQ, PM_ENV_CPPC_CPUFREQ};
 
 /**
  * @warning
-- 
2.20.1

[dpdk-dev] [PATCH v2 2/2] test/power: round cpuinfo cur freq only when using CPPC cpufreq

[Richael Zhuang]

On arm platform, the value in "/sys/.../cpuinfo_cur_freq" may not
be exactly the same as what was set when using CPPC cpufreq driver.
For other cpufreq driver, no need to round it currently, or else
this check will fail with turbo enabled. For example, with acpi_cpufreq,
cpuinfo_cur_freq can be 2401000 which is equal to freqs[0].It should
not be rounded to 2400000.

Fixes: 606a234c6d360 ("test/power: round CPU frequency to check")
Cc: richael.zhuangw@intel.com
Cc: stable@dpdk.org

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
---
 app/test/test_power_cpufreq.c | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index 87d07191d..0e195afc9 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -53,7 +53,9 @@ check_cur_freq(unsigned lcore_id, uint32_t idx)
 	FILE *f;
 	char fullpath[PATH_MAX];
 	char buf[BUFSIZ];
+	enum power_management_env env;
 	uint32_t cur_freq;
+	uint32_t freq_conv;
 	int ret = -1;
 	int i;
 
@@ -71,15 +73,18 @@ check_cur_freq(unsigned lcore_id, uint32_t idx)
 			goto fail_all;
 
 		cur_freq = strtoul(buf, NULL, TEST_POWER_CONVERT_TO_DECIMAL);
-
-		/* convert the frequency to nearest 100000 value
-		 * Ex: if cur_freq=1396789 then freq_conv=1400000
-		 * Ex: if cur_freq=800030 then freq_conv=800000
-		 */
-		unsigned int freq_conv = 0;
-		freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
-					/ TEST_ROUND_FREQ_TO_N_100000;
-		freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		freq_conv = cur_freq;
+
+		env = rte_power_get_env();
+		if (env == PM_ENV_CPPC_CPUFREQ) {
+			/* convert the frequency to nearest 100000 value
+			 * Ex: if cur_freq=1396789 then freq_conv=1400000
+			 * Ex: if cur_freq=800030 then freq_conv=800000
+			 */
+			freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
+						/ TEST_ROUND_FREQ_TO_N_100000;
+			freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		}
 
 		ret = (freqs[idx] == freq_conv ? 0 : -1);
 
-- 
2.20.1

[dpdk-dev] [PATCH v3 0/2] power: add support for cppc cpufreq driver

[Richael Zhuang]

v3:
rebase on Anatoly's patch: http://dpdk.org/patch/92077;
fix check_cur_freq bug: round cur freq only when using CPPC driver

Richael Zhuang (2):
  power: add support for cppc cpufreq
  test/power: round cpuinfo cur freq only when using CPPC cpufreq

 app/test/test_power.c          |   3 +-
 app/test/test_power_cpufreq.c  |  26 +-
 lib/power/meson.build          |   1 +
 lib/power/power_cppc_cpufreq.c | 632 +++++++++++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h | 229 ++++++++++++
 lib/power/rte_power.c          |  26 ++
 lib/power/rte_power.h          |   2 +-
 7 files changed, 907 insertions(+), 12 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

-- 
2.20.1

[dpdk-dev] [PATCH v3 1/2] power: add support for cppc cpufreq

[Richael Zhuang]

Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
supported, which are both not available on arm64 platforms. Add
support for cppc_cpufreq driver which works on most arm64 platforms.

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
---
 app/test/test_power.c          |   3 +-
 app/test/test_power_cpufreq.c  |   3 +-
 lib/power/meson.build          |   1 +
 lib/power/power_cppc_cpufreq.c | 632 +++++++++++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h | 229 ++++++++++++
 lib/power/rte_power.c          |  26 ++
 lib/power/rte_power.h          |   2 +-
 7 files changed, 893 insertions(+), 3 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

diff --git a/app/test/test_power.c b/app/test/test_power.c
index da1d67c0a..b7b556134 100644
--- a/app/test/test_power.c
+++ b/app/test/test_power.c
@@ -133,7 +133,8 @@ test_power(void)
 	/* Perform tests for valid environments.*/
 	const enum power_management_env envs[] = {PM_ENV_ACPI_CPUFREQ,
 			PM_ENV_KVM_VM,
-			PM_ENV_PSTATE_CPUFREQ};
+			PM_ENV_PSTATE_CPUFREQ,
+			PM_ENV_CPPC_CPUFREQ};
 
 	unsigned int i;
 	for (i = 0; i < RTE_DIM(envs); ++i) {
diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index f753d24ac..87d07191d 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -470,7 +470,8 @@ test_power_cpufreq(void)
 
 	/* Test environment configuration */
 	env = rte_power_get_env();
-	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ)) {
+	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ) &&
+			(env != PM_ENV_CPPC_CPUFREQ)) {
 		printf("Unexpectedly got an environment other than ACPI/PSTATE\n");
 		goto fail_all;
 	}
diff --git a/lib/power/meson.build b/lib/power/meson.build
index 74c5f3a29..4a5b07292 100644
--- a/lib/power/meson.build
+++ b/lib/power/meson.build
@@ -21,6 +21,7 @@ sources = files(
         'rte_power.c',
         'rte_power_empty_poll.c',
         'rte_power_pmd_mgmt.c',
+	'power_cppc_cpufreq.c',
 )
 headers = files(
         'rte_power.h',
diff --git a/lib/power/power_cppc_cpufreq.c b/lib/power/power_cppc_cpufreq.c
new file mode 100644
index 000000000..c4a71a3d3
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.c
@@ -0,0 +1,632 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#include <rte_memcpy.h>
+#include <rte_memory.h>
+
+#include "power_cppc_cpufreq.h"
+#include "power_common.h"
+
+/* macros used for rounding frequency to nearest 100000 */
+#define FREQ_ROUNDING_DELTA 50000
+#define ROUND_FREQ_TO_N_100000 100000
+
+/* the unit of highest_perf and nominal_perf differs on different arm platforms.
+ * For highest_perf, it maybe 300 or 3000000, both means 3.0GHz.
+ */
+#define UNIT_DIFF 10000
+
+#define POWER_CONVERT_TO_DECIMAL 10
+
+#define POWER_GOVERNOR_USERSPACE "userspace"
+#define POWER_SYSFILE_SETSPEED   \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_setspeed"
+#define POWER_SYSFILE_SCALING_MAX_FREQ \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
+#define POWER_SYSFILE_SCALING_MIN_FREQ  \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
+#define POWER_SYSFILE_HIGHEST_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/highest_perf"
+#define POWER_SYSFILE_NOMINAL_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/nominal_perf"
+#define POWER_SYSFILE_SYS_MAX \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
+
+#define POWER_CPPC_DRIVER "cppc-cpufreq"
+#define BUS_FREQ     100000
+
+enum power_state {
+	POWER_IDLE = 0,
+	POWER_ONGOING,
+	POWER_USED,
+	POWER_UNKNOWN
+};
+
+/**
+ * Power info per lcore.
+ */
+struct cppc_power_info {
+	unsigned int lcore_id;                   /**< Logical core id */
+	uint32_t state;                      /**< Power in use state */
+	FILE *f;                             /**< FD of scaling_setspeed */
+	char governor_ori[32];               /**< Original governor name */
+	uint32_t curr_idx;                   /**< Freq index in freqs array */
+	uint32_t highest_perf;		     /**< system wide max freq */
+	uint32_t nominal_perf;		     /**< system wide nominal freq */
+	uint16_t turbo_available;            /**< Turbo Boost available */
+	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
+	uint32_t nb_freqs;                   /**< number of available freqs */
+	uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
+} __rte_cache_aligned;
+
+static struct cppc_power_info lcore_power_info[RTE_MAX_LCORE];
+
+/**
+ * It is to set specific freq for specific logical core, according to the index
+ * of supported frequencies.
+ */
+static int
+set_freq_internal(struct cppc_power_info *pi, uint32_t idx)
+{
+	if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
+				"should be less than %u\n", idx, pi->nb_freqs);
+		return -1;
+	}
+
+	/* Check if it is the same as current */
+	if (idx == pi->curr_idx)
+		return 0;
+
+	POWER_DEBUG_TRACE("Frequency[%u] %u to be set for lcore %u\n",
+			idx, pi->freqs[idx], pi->lcore_id);
+	if (fseek(pi->f, 0, SEEK_SET) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
+				"for setting frequency for lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	if (fprintf(pi->f, "%u", pi->freqs[idx]) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new frequency for "
+				"lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	fflush(pi->f);
+	pi->curr_idx = idx;
+
+	return 1;
+}
+
+/**
+ * It is to check the current scaling governor by reading sys file, and then
+ * set it into 'userspace' if it is not by writing the sys file. The original
+ * governor will be saved for rolling back.
+ */
+static int
+power_set_governor_userspace(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, POWER_GOVERNOR_USERSPACE,
+			pi->governor_ori, sizeof(pi->governor_ori));
+}
+
+static int
+power_check_turbo(struct cppc_power_info *pi)
+{
+	FILE *f_nom, *f_max, *f_cmax;
+	int ret = -1;
+	uint32_t nominal_perf = 0, highest_perf = 0, cpuinfo_max_freq = 0;
+
+	open_core_sysfs_file(POWER_SYSFILE_HIGHEST_PERF, pi->lcore_id, "r",
+			&f_max);
+	FOPEN_OR_ERR_GOTO(f_max, err);
+
+	open_core_sysfs_file(POWER_SYSFILE_NOMINAL_PERF, pi->lcore_id, "r",
+			&f_nom);
+	FOPEN_OR_ERR_GOTO(f_nom, err);
+
+	open_core_sysfs_file(POWER_SYSFILE_SYS_MAX, pi->lcore_id, "r",
+			&f_cmax);
+	FOPEN_OR_ERR_GOTO(f_cmax, err);
+
+	ret = read_core_sysfs_u32(f_max, &highest_perf);
+	FOPS_OR_ERR_GOTO(ret, err);
+
+	ret = read_core_sysfs_u32(f_nom, &nominal_perf);
+	FOPS_OR_ERR_GOTO(ret, err);
+
+	ret = read_core_sysfs_u32(f_cmax, &cpuinfo_max_freq);
+	FOPS_OR_ERR_GOTO(ret, err);
+
+	pi->highest_perf = highest_perf;
+	pi->nominal_perf = nominal_perf;
+
+	if ((highest_perf > nominal_perf) && ((cpuinfo_max_freq == highest_perf) ||
+				cpuinfo_max_freq == highest_perf * UNIT_DIFF)) {
+		pi->turbo_available = 1;
+		pi->turbo_enable = 1;
+		ret = 0;
+		POWER_DEBUG_TRACE("Lcore %u can do Turbo Boost! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	} else {
+		pi->turbo_available = 0;
+		pi->turbo_enable = 0;
+		POWER_DEBUG_TRACE("Lcore %u Turbo not available! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	}
+
+err:
+	if (f_max != NULL)
+		fclose(f_max);
+	if (f_nom != NULL)
+		fclose(f_nom);
+	if (f_cmax != NULL)
+		fclose(f_cmax);
+
+	return ret;
+}
+
+/**
+ * It is to get the available frequencies of the specific lcore by reading the
+ * sys file.
+ */
+static int
+power_get_available_freqs(struct cppc_power_info *pi)
+{
+	FILE *f_min, *f_max;
+	int ret = -1;
+	uint32_t scaling_min_freq = 0, scaling_max_freq = 0, nominal_perf = 0;
+	uint32_t i, num_freqs = 0;
+
+	open_core_sysfs_file(POWER_SYSFILE_SCALING_MAX_FREQ, pi->lcore_id, "r",
+			&f_max);
+	FOPEN_OR_ERR_GOTO(f_max, out);
+
+	open_core_sysfs_file(POWER_SYSFILE_SCALING_MIN_FREQ, pi->lcore_id, "r",
+			&f_min);
+	FOPEN_OR_ERR_GOTO(f_min, out);
+
+	ret = read_core_sysfs_u32(f_max, &scaling_max_freq);
+	FOPS_OR_ERR_GOTO(ret, out);
+
+	ret = read_core_sysfs_u32(f_min, &scaling_min_freq);
+	FOPS_OR_ERR_GOTO(ret, out);
+
+	power_check_turbo(pi);
+
+	if (scaling_max_freq < scaling_min_freq)
+		goto out;
+
+	/* If turbo is available then there is one extra freq bucket
+	 * to store the sys max freq which value is scaling_max_freq
+	 */
+	nominal_perf = (pi->nominal_perf < UNIT_DIFF) ?
+			pi->nominal_perf * UNIT_DIFF : pi->nominal_perf;
+	num_freqs = (nominal_perf - scaling_min_freq) / BUS_FREQ + 1 +
+		pi->turbo_available;
+
+	/* Generate the freq bucket array. */
+	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
+		if ((i == 0) && pi->turbo_available)
+			pi->freqs[pi->nb_freqs++] = scaling_max_freq;
+		else
+			pi->freqs[pi->nb_freqs++] =
+			nominal_perf - (i - pi->turbo_available) * BUS_FREQ;
+	}
+
+	ret = 0;
+
+	POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
+			num_freqs, pi->lcore_id);
+
+out:
+	if (f_min != NULL)
+		fclose(f_min);
+	if (f_max != NULL)
+		fclose(f_max);
+
+	return ret;
+}
+
+/**
+ * It is to fopen the sys file for the future setting the lcore frequency.
+ */
+static int
+power_init_for_setting_freq(struct cppc_power_info *pi)
+{
+	FILE *f;
+	char buf[BUFSIZ];
+	uint32_t i, freq;
+	int ret;
+
+	open_core_sysfs_file(POWER_SYSFILE_SETSPEED, pi->lcore_id, "rw+",
+			&f);
+	FOPEN_OR_ERR_GOTO(f, err);
+
+	ret = read_core_sysfs_s(f, buf, sizeof(buf));
+	FOPS_OR_ERR_GOTO(ret, err);
+
+	freq = strtoul(buf, NULL, POWER_CONVERT_TO_DECIMAL);
+
+	/* convert the frequency to nearest 100000 value
+	 * Ex: if freq=1396789 then freq_conv=1400000
+	 * Ex: if freq=800030 then freq_conv=800000
+	 */
+	unsigned int freq_conv = 0;
+	freq_conv = (freq + FREQ_ROUNDING_DELTA)
+				/ ROUND_FREQ_TO_N_100000;
+	freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
+
+	for (i = 0; i < pi->nb_freqs; i++) {
+		if (freq_conv == pi->freqs[i]) {
+			pi->curr_idx = i;
+			pi->f = f;
+			return 0;
+		}
+	}
+
+err:
+	if (f != NULL)
+		fclose(f);
+
+	return -1;
+}
+
+int
+power_cppc_cpufreq_check_supported(void)
+{
+	return cpufreq_check_scaling_driver(POWER_CPPC_DRIVER);
+}
+
+int
+power_cppc_cpufreq_init(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_IDLE;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"in use\n", lcore_id);
+		return -1;
+	}
+
+	pi->lcore_id = lcore_id;
+	/* Check and set the governor */
+	if (power_set_governor_userspace(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
+				"userspace\n", lcore_id);
+		goto fail;
+	}
+
+	/* Get the available frequencies */
+	if (power_get_available_freqs(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Init for setting lcore frequency */
+	if (power_init_for_setting_freq(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Set freq to max by default */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
+				"to max\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
+			"power management\n", lcore_id);
+
+	__atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	return -1;
+}
+
+/**
+ * It is to check the governor and then set the original governor back if
+ * needed by writing the sys file.
+ */
+static int
+power_set_governor_original(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, pi->governor_ori, NULL, 0);
+}
+
+int
+power_cppc_cpufreq_exit(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_USED;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"not used\n", lcore_id);
+		return -1;
+	}
+
+	/* Close FD of setting freq */
+	fclose(pi->f);
+	pi->f = NULL;
+
+	/* Set the governor back to the original */
+	if (power_set_governor_original(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
+				"to the original\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
+			"'userspace' mode and been set back to the "
+			"original\n", lcore_id);
+	__atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+
+	return -1;
+}
+
+uint32_t
+power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return 0;
+	}
+
+	if (freqs == NULL) {
+		RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
+		return 0;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (num < pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
+		return 0;
+	}
+	rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
+
+	return pi->nb_freqs;
+}
+
+uint32_t
+power_cppc_cpufreq_get_freq(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return RTE_POWER_INVALID_FREQ_INDEX;
+	}
+
+	return lcore_power_info[lcore_id].curr_idx;
+}
+
+int
+power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	return set_freq_internal(&(lcore_power_info[lcore_id]), index);
+}
+
+int
+power_cppc_cpufreq_freq_down(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx + 1 == pi->nb_freqs)
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx + 1);
+}
+
+int
+power_cppc_cpufreq_freq_up(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx == 0 || (pi->curr_idx == 1 &&
+		pi->turbo_available && !pi->turbo_enable))
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx - 1);
+}
+
+int
+power_cppc_cpufreq_freq_max(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	/* Frequencies in the array are from high to low. */
+	if (lcore_power_info[lcore_id].turbo_available) {
+		if (lcore_power_info[lcore_id].turbo_enable)
+			/* Set to Turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 0);
+		else
+			/* Set to max non-turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 1);
+	} else
+		return set_freq_internal(&lcore_power_info[lcore_id], 0);
+}
+
+int
+power_cppc_cpufreq_freq_min(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->nb_freqs - 1);
+}
+
+int
+power_cppc_turbo_status(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	return pi->turbo_enable;
+}
+
+int
+power_cppc_enable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	if (pi->turbo_available)
+		pi->turbo_enable = 1;
+	else {
+		pi->turbo_enable = 0;
+		RTE_LOG(ERR, POWER,
+			"Failed to enable turbo on lcore %u\n",
+			lcore_id);
+		return -1;
+	}
+
+	/* TODO: must set to max once enbling Turbo? Considering add condition:
+	 * if ((pi->turbo_available) && (pi->curr_idx <= 1))
+	 */
+	/* Max may have changed, so call to max function */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER,
+			"Failed to set frequency of lcore %u to max\n",
+			lcore_id);
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+power_cppc_disable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	pi->turbo_enable = 0;
+
+	if ((pi->turbo_available) && (pi->curr_idx <= 1)) {
+		/* Try to set freq to max by default coming out of turbo */
+		if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+			RTE_LOG(ERR, POWER,
+				"Failed to set frequency of lcore %u to max\n",
+				lcore_id);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+int
+power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+	if (caps == NULL) {
+		RTE_LOG(ERR, POWER, "Invalid argument\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	caps->capabilities = 0;
+	caps->turbo = !!(pi->turbo_available);
+
+	return 0;
+}
diff --git a/lib/power/power_cppc_cpufreq.h b/lib/power/power_cppc_cpufreq.h
new file mode 100644
index 000000000..427ce4f83
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.h
@@ -0,0 +1,229 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#ifndef _POWER_CPPC_CPUFREQ_H
+#define _POWER_CPPC_CPUFREQ_H
+
+/**
+ * @file
+ * RTE Power Management via userspace CPPC cpufreq
+ */
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_string_fns.h>
+#include "rte_power.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Check if CPPC power management is supported.
+ *
+ * @return
+ *   - 1 if supported
+ *   - 0 if unsupported
+ *   - -1 if error, with rte_errno indicating reason for error.
+ */
+int power_cppc_cpufreq_check_supported(void);
+
+/**
+ * Initialize power management for a specific lcore. It will check and set the
+ * governor to userspace for the lcore, get the available frequencies, and
+ * prepare to set new lcore frequency.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_init(unsigned int lcore_id);
+
+/**
+ * Exit power management on a specific lcore. It will set the governor to which
+ * is before initialized.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_exit(unsigned int lcore_id);
+
+/**
+ * Get the available frequencies of a specific lcore. The return value will be
+ * the minimal one of the total number of available frequencies and the number
+ * of buffer. The index of available frequencies used in other interfaces
+ * should be in the range of 0 to this return value.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param freqs
+ *  The buffer array to save the frequencies.
+ * @param num
+ *  The number of frequencies to get.
+ *
+ * @return
+ *  The number of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs,
+		uint32_t num);
+
+/**
+ * Return the current index of available frequencies of a specific lcore. It
+ * will return 'RTE_POWER_INVALID_FREQ_INDEX = (~0)' if error.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  The current index of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_get_freq(unsigned int lcore_id);
+
+/**
+ * Set the new frequency for a specific lcore by indicating the index of
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param index
+ *  The index of available frequencies.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index);
+
+/**
+ * Scale up the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_up(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_down(unsigned int lcore_id);
+
+/**
+ * Scale up the frequency of a specific lcore to the highest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_max(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore to the lowest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_min(unsigned int lcore_id);
+
+/**
+ * Get the turbo status of a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 Turbo Boost is enabled on this lcore.
+ *  - 0 Turbo Boost is disabled on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_turbo_status(unsigned int lcore_id);
+
+/**
+ * Enable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost is enabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_enable_turbo(unsigned int lcore_id);
+
+/**
+ * Disable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost disabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_disable_turbo(unsigned int lcore_id);
+
+/**
+ * Returns power capabilities for a specific lcore.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param caps
+ *  pointer to rte_power_core_capabilities object.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/power/rte_power.c b/lib/power/rte_power.c
index 98eaba915..3d51ff8be 100644
--- a/lib/power/rte_power.c
+++ b/lib/power/rte_power.c
@@ -10,6 +10,7 @@
 #include "power_kvm_vm.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
+#include "power_cppc_cpufreq.h"
 
 enum power_management_env global_default_env = PM_ENV_NOT_SET;
 
@@ -54,6 +55,8 @@ rte_power_check_env_supported(enum power_management_env env)
 		return power_pstate_cpufreq_check_supported();
 	case PM_ENV_KVM_VM:
 		return power_kvm_vm_check_supported();
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_check_supported();
 	default:
 		rte_errno = EINVAL;
 		return -1;
@@ -110,6 +113,18 @@ rte_power_set_env(enum power_management_env env)
 		rte_power_freq_disable_turbo = power_pstate_disable_turbo;
 		rte_power_get_capabilities = power_pstate_get_capabilities;
 
+	} else if (env == PM_ENV_CPPC_CPUFREQ) {
+		rte_power_freqs = power_cppc_cpufreq_freqs;
+		rte_power_get_freq = power_cppc_cpufreq_get_freq;
+		rte_power_set_freq = power_cppc_cpufreq_set_freq;
+		rte_power_freq_up = power_cppc_cpufreq_freq_up;
+		rte_power_freq_down = power_cppc_cpufreq_freq_down;
+		rte_power_freq_min = power_cppc_cpufreq_freq_min;
+		rte_power_freq_max = power_cppc_cpufreq_freq_max;
+		rte_power_turbo_status = power_cppc_turbo_status;
+		rte_power_freq_enable_turbo = power_cppc_enable_turbo;
+		rte_power_freq_disable_turbo = power_cppc_disable_turbo;
+		rte_power_get_capabilities = power_cppc_get_capabilities;
 	} else {
 		RTE_LOG(ERR, POWER, "Invalid Power Management Environment(%d) set\n",
 				env);
@@ -153,6 +168,8 @@ rte_power_init(unsigned int lcore_id)
 		return power_kvm_vm_init(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_init(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_init(lcore_id);
 	default:
 		RTE_LOG(INFO, POWER, "Env isn't set yet!\n");
 	}
@@ -172,6 +189,13 @@ rte_power_init(unsigned int lcore_id)
 		goto out;
 	}
 
+	RTE_LOG(INFO, POWER, "Attempting to initialise CPPC power management...\n");
+	ret = power_cppc_cpufreq_init(lcore_id);
+	if (ret == 0) {
+		rte_power_set_env(PM_ENV_CPPC_CPUFREQ);
+		goto out;
+	}
+
 	RTE_LOG(INFO, POWER, "Attempting to initialise VM power management...\n");
 	ret = power_kvm_vm_init(lcore_id);
 	if (ret == 0) {
@@ -194,6 +218,8 @@ rte_power_exit(unsigned int lcore_id)
 		return power_kvm_vm_exit(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_exit(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_exit(lcore_id);
 	default:
 		RTE_LOG(ERR, POWER, "Environment has not been set, unable to exit gracefully\n");
 
diff --git a/lib/power/rte_power.h b/lib/power/rte_power.h
index c8086bf6b..c5759afa3 100644
--- a/lib/power/rte_power.h
+++ b/lib/power/rte_power.h
@@ -22,7 +22,7 @@ extern "C" {
 
 /* Power Management Environment State */
 enum power_management_env {PM_ENV_NOT_SET, PM_ENV_ACPI_CPUFREQ, PM_ENV_KVM_VM,
-		PM_ENV_PSTATE_CPUFREQ};
+		PM_ENV_PSTATE_CPUFREQ, PM_ENV_CPPC_CPUFREQ};
 
 /**
  * @warning
-- 
2.20.1

[dpdk-dev] [PATCH v4 0/2] power: add support for cppc cpufreq driver

[Richael Zhuang]

v4:
rebase on Anatoly's patch: http://dpdk.org/patch/94676

Richael Zhuang (2):
  power: add support for cppc cpufreq
  test/power: round cpuinfo cur freq only when using  CPPC cpufreq

 app/test/test_power.c          |   3 +-
 app/test/test_power_cpufreq.c  |  26 +-
 lib/power/meson.build          |   1 +
 lib/power/power_cppc_cpufreq.c | 632 +++++++++++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h | 229 ++++++++++++
 lib/power/rte_power.c          |  26 ++
 lib/power/rte_power.h          |   2 +-
 7 files changed, 907 insertions(+), 12 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

-- 
2.20.1

[dpdk-dev] [PATCH v4 1/2] power: add support for cppc cpufreq

[Richael Zhuang]

Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
supported, which are both not available on arm64 platforms. Add
support for cppc_cpufreq driver which works on most arm64 platforms.

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
---
 app/test/test_power.c          |   3 +-
 app/test/test_power_cpufreq.c  |   3 +-
 lib/power/meson.build          |   1 +
 lib/power/power_cppc_cpufreq.c | 632 +++++++++++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h | 229 ++++++++++++
 lib/power/rte_power.c          |  26 ++
 lib/power/rte_power.h          |   2 +-
 7 files changed, 893 insertions(+), 3 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

diff --git a/app/test/test_power.c b/app/test/test_power.c
index da1d67c0a..b7b556134 100644
--- a/app/test/test_power.c
+++ b/app/test/test_power.c
@@ -133,7 +133,8 @@ test_power(void)
 	/* Perform tests for valid environments.*/
 	const enum power_management_env envs[] = {PM_ENV_ACPI_CPUFREQ,
 			PM_ENV_KVM_VM,
-			PM_ENV_PSTATE_CPUFREQ};
+			PM_ENV_PSTATE_CPUFREQ,
+			PM_ENV_CPPC_CPUFREQ};
 
 	unsigned int i;
 	for (i = 0; i < RTE_DIM(envs); ++i) {
diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index 0c3adc5f3..8516df4ca 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -496,7 +496,8 @@ test_power_cpufreq(void)
 
 	/* Test environment configuration */
 	env = rte_power_get_env();
-	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ)) {
+	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ) &&
+			(env != PM_ENV_CPPC_CPUFREQ)) {
 		printf("Unexpectedly got an environment other than ACPI/PSTATE\n");
 		goto fail_all;
 	}
diff --git a/lib/power/meson.build b/lib/power/meson.build
index 74c5f3a29..4a5b07292 100644
--- a/lib/power/meson.build
+++ b/lib/power/meson.build
@@ -21,6 +21,7 @@ sources = files(
         'rte_power.c',
         'rte_power_empty_poll.c',
         'rte_power_pmd_mgmt.c',
+	'power_cppc_cpufreq.c',
 )
 headers = files(
         'rte_power.h',
diff --git a/lib/power/power_cppc_cpufreq.c b/lib/power/power_cppc_cpufreq.c
new file mode 100644
index 000000000..fd4483e52
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.c
@@ -0,0 +1,632 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#include <rte_memcpy.h>
+#include <rte_memory.h>
+
+#include "power_cppc_cpufreq.h"
+#include "power_common.h"
+
+/* macros used for rounding frequency to nearest 100000 */
+#define FREQ_ROUNDING_DELTA 50000
+#define ROUND_FREQ_TO_N_100000 100000
+
+/* the unit of highest_perf and nominal_perf differs on different arm platforms.
+ * For highest_perf, it maybe 300 or 3000000, both means 3.0GHz.
+ */
+#define UNIT_DIFF 10000
+
+#define POWER_CONVERT_TO_DECIMAL 10
+
+#define POWER_GOVERNOR_USERSPACE "userspace"
+#define POWER_SYSFILE_SETSPEED   \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_setspeed"
+#define POWER_SYSFILE_SCALING_MAX_FREQ \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
+#define POWER_SYSFILE_SCALING_MIN_FREQ  \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
+#define POWER_SYSFILE_HIGHEST_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/highest_perf"
+#define POWER_SYSFILE_NOMINAL_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/nominal_perf"
+#define POWER_SYSFILE_SYS_MAX \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
+
+#define POWER_CPPC_DRIVER "cppc-cpufreq"
+#define BUS_FREQ     100000
+
+enum power_state {
+	POWER_IDLE = 0,
+	POWER_ONGOING,
+	POWER_USED,
+	POWER_UNKNOWN
+};
+
+/**
+ * Power info per lcore.
+ */
+struct cppc_power_info {
+	unsigned int lcore_id;                   /**< Logical core id */
+	uint32_t state;                      /**< Power in use state */
+	FILE *f;                             /**< FD of scaling_setspeed */
+	char governor_ori[32];               /**< Original governor name */
+	uint32_t curr_idx;                   /**< Freq index in freqs array */
+	uint32_t highest_perf;		     /**< system wide max freq */
+	uint32_t nominal_perf;		     /**< system wide nominal freq */
+	uint16_t turbo_available;            /**< Turbo Boost available */
+	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
+	uint32_t nb_freqs;                   /**< number of available freqs */
+	uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
+} __rte_cache_aligned;
+
+static struct cppc_power_info lcore_power_info[RTE_MAX_LCORE];
+
+/**
+ * It is to set specific freq for specific logical core, according to the index
+ * of supported frequencies.
+ */
+static int
+set_freq_internal(struct cppc_power_info *pi, uint32_t idx)
+{
+	if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
+				"should be less than %u\n", idx, pi->nb_freqs);
+		return -1;
+	}
+
+	/* Check if it is the same as current */
+	if (idx == pi->curr_idx)
+		return 0;
+
+	POWER_DEBUG_TRACE("Frequency[%u] %u to be set for lcore %u\n",
+			idx, pi->freqs[idx], pi->lcore_id);
+	if (fseek(pi->f, 0, SEEK_SET) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
+			"for setting frequency for lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	if (fprintf(pi->f, "%u", pi->freqs[idx]) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new frequency for "
+				"lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	fflush(pi->f);
+	pi->curr_idx = idx;
+
+	return 1;
+}
+
+/**
+ * It is to check the current scaling governor by reading sys file, and then
+ * set it into 'userspace' if it is not by writing the sys file. The original
+ * governor will be saved for rolling back.
+ */
+static int
+power_set_governor_userspace(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, POWER_GOVERNOR_USERSPACE,
+			pi->governor_ori, sizeof(pi->governor_ori));
+}
+
+static int
+power_check_turbo(struct cppc_power_info *pi)
+{
+	FILE *f_nom, *f_max, *f_cmax;
+	int ret = -1;
+	uint32_t nominal_perf = 0, highest_perf = 0, cpuinfo_max_freq = 0;
+
+	open_core_sysfs_file(POWER_SYSFILE_HIGHEST_PERF, pi->lcore_id, "r",
+			&f_max);
+	FOPEN_OR_ERR_GOTO(f_max, err);
+
+	open_core_sysfs_file(POWER_SYSFILE_NOMINAL_PERF, pi->lcore_id, "r",
+			&f_nom);
+	FOPEN_OR_ERR_GOTO(f_nom, err);
+
+	open_core_sysfs_file(POWER_SYSFILE_SYS_MAX, pi->lcore_id, "r",
+			&f_cmax);
+	FOPEN_OR_ERR_GOTO(f_cmax, err);
+
+	ret = read_core_sysfs_u32(f_max, &highest_perf);
+	FOPS_OR_ERR_GOTO(ret, err);
+
+	ret = read_core_sysfs_u32(f_nom, &nominal_perf);
+	FOPS_OR_ERR_GOTO(ret, err);
+
+	ret = read_core_sysfs_u32(f_cmax, &cpuinfo_max_freq);
+	FOPS_OR_ERR_GOTO(ret, err);
+
+	pi->highest_perf = highest_perf;
+	pi->nominal_perf = nominal_perf;
+
+	if ((highest_perf > nominal_perf) && ((cpuinfo_max_freq == highest_perf) ||
+				cpuinfo_max_freq == highest_perf * UNIT_DIFF)) {
+		pi->turbo_available = 1;
+		pi->turbo_enable = 1;
+		ret = 0;
+		POWER_DEBUG_TRACE("Lcore %u can do Turbo Boost! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	} else {
+		pi->turbo_available = 0;
+		pi->turbo_enable = 0;
+		POWER_DEBUG_TRACE("Lcore %u Turbo not available! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	}
+
+err:
+	if (f_max != NULL)
+		fclose(f_max);
+	if (f_nom != NULL)
+		fclose(f_nom);
+	if (f_cmax != NULL)
+		fclose(f_cmax);
+
+	return ret;
+}
+
+/**
+ * It is to get the available frequencies of the specific lcore by reading the
+ * sys file.
+ */
+static int
+power_get_available_freqs(struct cppc_power_info *pi)
+{
+	FILE *f_min, *f_max;
+	int ret = -1;
+	uint32_t scaling_min_freq = 0, scaling_max_freq = 0, nominal_perf = 0;
+	uint32_t i, num_freqs = 0;
+
+	open_core_sysfs_file(POWER_SYSFILE_SCALING_MAX_FREQ, pi->lcore_id, "r",
+			&f_max);
+	FOPEN_OR_ERR_GOTO(f_max, out);
+
+	open_core_sysfs_file(POWER_SYSFILE_SCALING_MIN_FREQ, pi->lcore_id, "r",
+			&f_min);
+	FOPEN_OR_ERR_GOTO(f_min, out);
+
+	ret = read_core_sysfs_u32(f_max, &scaling_max_freq);
+	FOPS_OR_ERR_GOTO(ret, out);
+
+	ret = read_core_sysfs_u32(f_min, &scaling_min_freq);
+	FOPS_OR_ERR_GOTO(ret, out);
+
+	power_check_turbo(pi);
+
+	if (scaling_max_freq < scaling_min_freq)
+		goto out;
+
+	/* If turbo is available then there is one extra freq bucket
+	 * to store the sys max freq which value is scaling_max_freq
+	 */
+	nominal_perf = (pi->nominal_perf < UNIT_DIFF) ?
+			pi->nominal_perf * UNIT_DIFF : pi->nominal_perf;
+	num_freqs = (nominal_perf - scaling_min_freq) / BUS_FREQ + 1 +
+		pi->turbo_available;
+
+	/* Generate the freq bucket array. */
+	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
+		if ((i == 0) && pi->turbo_available)
+			pi->freqs[pi->nb_freqs++] = scaling_max_freq;
+		else
+			pi->freqs[pi->nb_freqs++] =
+			nominal_perf - (i - pi->turbo_available) * BUS_FREQ;
+	}
+
+	ret = 0;
+
+	POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
+			num_freqs, pi->lcore_id);
+
+out:
+	if (f_min != NULL)
+		fclose(f_min);
+	if (f_max != NULL)
+		fclose(f_max);
+
+	return ret;
+}
+
+/**
+ * It is to fopen the sys file for the future setting the lcore frequency.
+ */
+static int
+power_init_for_setting_freq(struct cppc_power_info *pi)
+{
+	FILE *f;
+	char buf[BUFSIZ];
+	uint32_t i, freq;
+	int ret;
+
+	open_core_sysfs_file(POWER_SYSFILE_SETSPEED, pi->lcore_id, "rw+",
+			&f);
+	FOPEN_OR_ERR_GOTO(f, err);
+
+	ret = read_core_sysfs_s(f, buf, sizeof(buf));
+	FOPS_OR_ERR_GOTO(ret, err);
+
+	freq = strtoul(buf, NULL, POWER_CONVERT_TO_DECIMAL);
+
+	/* convert the frequency to nearest 100000 value
+	 * Ex: if freq=1396789 then freq_conv=1400000
+	 * Ex: if freq=800030 then freq_conv=800000
+	 */
+	unsigned int freq_conv = 0;
+	freq_conv = (freq + FREQ_ROUNDING_DELTA)
+				/ ROUND_FREQ_TO_N_100000;
+	freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
+
+	for (i = 0; i < pi->nb_freqs; i++) {
+		if (freq_conv == pi->freqs[i]) {
+			pi->curr_idx = i;
+			pi->f = f;
+			return 0;
+		}
+	}
+
+err:
+	if (f != NULL)
+		fclose(f);
+
+	return -1;
+}
+
+int
+power_cppc_cpufreq_check_supported(void)
+{
+	return cpufreq_check_scaling_driver(POWER_CPPC_DRIVER);
+}
+
+int
+power_cppc_cpufreq_init(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_IDLE;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"in use\n", lcore_id);
+		return -1;
+	}
+
+	pi->lcore_id = lcore_id;
+	/* Check and set the governor */
+	if (power_set_governor_userspace(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
+				"userspace\n", lcore_id);
+		goto fail;
+	}
+
+	/* Get the available frequencies */
+	if (power_get_available_freqs(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Init for setting lcore frequency */
+	if (power_init_for_setting_freq(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Set freq to max by default */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
+				"to max\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
+			"power management\n", lcore_id);
+
+	__atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	return -1;
+}
+
+/**
+ * It is to check the governor and then set the original governor back if
+ * needed by writing the sys file.
+ */
+static int
+power_set_governor_original(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, pi->governor_ori, NULL, 0);
+}
+
+int
+power_cppc_cpufreq_exit(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_USED;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"not used\n", lcore_id);
+		return -1;
+	}
+
+	/* Close FD of setting freq */
+	fclose(pi->f);
+	pi->f = NULL;
+
+	/* Set the governor back to the original */
+	if (power_set_governor_original(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
+				"to the original\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
+			"'userspace' mode and been set back to the "
+			"original\n", lcore_id);
+	__atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+
+	return -1;
+}
+
+uint32_t
+power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return 0;
+	}
+
+	if (freqs == NULL) {
+		RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
+		return 0;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (num < pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
+		return 0;
+	}
+	rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
+
+	return pi->nb_freqs;
+}
+
+uint32_t
+power_cppc_cpufreq_get_freq(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return RTE_POWER_INVALID_FREQ_INDEX;
+	}
+
+	return lcore_power_info[lcore_id].curr_idx;
+}
+
+int
+power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	return set_freq_internal(&(lcore_power_info[lcore_id]), index);
+}
+
+int
+power_cppc_cpufreq_freq_down(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx + 1 == pi->nb_freqs)
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx + 1);
+}
+
+int
+power_cppc_cpufreq_freq_up(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx == 0 || (pi->curr_idx == 1 &&
+		pi->turbo_available && !pi->turbo_enable))
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx - 1);
+}
+
+int
+power_cppc_cpufreq_freq_max(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	/* Frequencies in the array are from high to low. */
+	if (lcore_power_info[lcore_id].turbo_available) {
+		if (lcore_power_info[lcore_id].turbo_enable)
+			/* Set to Turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 0);
+		else
+			/* Set to max non-turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 1);
+	} else
+		return set_freq_internal(&lcore_power_info[lcore_id], 0);
+}
+
+int
+power_cppc_cpufreq_freq_min(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->nb_freqs - 1);
+}
+
+int
+power_cppc_turbo_status(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	return pi->turbo_enable;
+}
+
+int
+power_cppc_enable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	if (pi->turbo_available)
+		pi->turbo_enable = 1;
+	else {
+		pi->turbo_enable = 0;
+		RTE_LOG(ERR, POWER,
+			"Failed to enable turbo on lcore %u\n",
+			lcore_id);
+		return -1;
+	}
+
+	/* TODO: must set to max once enbling Turbo? Considering add condition:
+	 * if ((pi->turbo_available) && (pi->curr_idx <= 1))
+	 */
+	/* Max may have changed, so call to max function */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER,
+			"Failed to set frequency of lcore %u to max\n",
+			lcore_id);
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+power_cppc_disable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	pi->turbo_enable = 0;
+
+	if ((pi->turbo_available) && (pi->curr_idx <= 1)) {
+		/* Try to set freq to max by default coming out of turbo */
+		if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+			RTE_LOG(ERR, POWER,
+				"Failed to set frequency of lcore %u to max\n",
+				lcore_id);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+int
+power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+	if (caps == NULL) {
+		RTE_LOG(ERR, POWER, "Invalid argument\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	caps->capabilities = 0;
+	caps->turbo = !!(pi->turbo_available);
+
+	return 0;
+}
diff --git a/lib/power/power_cppc_cpufreq.h b/lib/power/power_cppc_cpufreq.h
new file mode 100644
index 000000000..427ce4f83
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.h
@@ -0,0 +1,229 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#ifndef _POWER_CPPC_CPUFREQ_H
+#define _POWER_CPPC_CPUFREQ_H
+
+/**
+ * @file
+ * RTE Power Management via userspace CPPC cpufreq
+ */
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_string_fns.h>
+#include "rte_power.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Check if CPPC power management is supported.
+ *
+ * @return
+ *   - 1 if supported
+ *   - 0 if unsupported
+ *   - -1 if error, with rte_errno indicating reason for error.
+ */
+int power_cppc_cpufreq_check_supported(void);
+
+/**
+ * Initialize power management for a specific lcore. It will check and set the
+ * governor to userspace for the lcore, get the available frequencies, and
+ * prepare to set new lcore frequency.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_init(unsigned int lcore_id);
+
+/**
+ * Exit power management on a specific lcore. It will set the governor to which
+ * is before initialized.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_exit(unsigned int lcore_id);
+
+/**
+ * Get the available frequencies of a specific lcore. The return value will be
+ * the minimal one of the total number of available frequencies and the number
+ * of buffer. The index of available frequencies used in other interfaces
+ * should be in the range of 0 to this return value.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param freqs
+ *  The buffer array to save the frequencies.
+ * @param num
+ *  The number of frequencies to get.
+ *
+ * @return
+ *  The number of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs,
+		uint32_t num);
+
+/**
+ * Return the current index of available frequencies of a specific lcore. It
+ * will return 'RTE_POWER_INVALID_FREQ_INDEX = (~0)' if error.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  The current index of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_get_freq(unsigned int lcore_id);
+
+/**
+ * Set the new frequency for a specific lcore by indicating the index of
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param index
+ *  The index of available frequencies.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index);
+
+/**
+ * Scale up the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_up(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_down(unsigned int lcore_id);
+
+/**
+ * Scale up the frequency of a specific lcore to the highest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_max(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore to the lowest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_min(unsigned int lcore_id);
+
+/**
+ * Get the turbo status of a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 Turbo Boost is enabled on this lcore.
+ *  - 0 Turbo Boost is disabled on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_turbo_status(unsigned int lcore_id);
+
+/**
+ * Enable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost is enabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_enable_turbo(unsigned int lcore_id);
+
+/**
+ * Disable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost disabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_disable_turbo(unsigned int lcore_id);
+
+/**
+ * Returns power capabilities for a specific lcore.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param caps
+ *  pointer to rte_power_core_capabilities object.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/power/rte_power.c b/lib/power/rte_power.c
index 98eaba915..3d51ff8be 100644
--- a/lib/power/rte_power.c
+++ b/lib/power/rte_power.c
@@ -10,6 +10,7 @@
 #include "power_kvm_vm.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
+#include "power_cppc_cpufreq.h"
 
 enum power_management_env global_default_env = PM_ENV_NOT_SET;
 
@@ -54,6 +55,8 @@ rte_power_check_env_supported(enum power_management_env env)
 		return power_pstate_cpufreq_check_supported();
 	case PM_ENV_KVM_VM:
 		return power_kvm_vm_check_supported();
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_check_supported();
 	default:
 		rte_errno = EINVAL;
 		return -1;
@@ -110,6 +113,18 @@ rte_power_set_env(enum power_management_env env)
 		rte_power_freq_disable_turbo = power_pstate_disable_turbo;
 		rte_power_get_capabilities = power_pstate_get_capabilities;
 
+	} else if (env == PM_ENV_CPPC_CPUFREQ) {
+		rte_power_freqs = power_cppc_cpufreq_freqs;
+		rte_power_get_freq = power_cppc_cpufreq_get_freq;
+		rte_power_set_freq = power_cppc_cpufreq_set_freq;
+		rte_power_freq_up = power_cppc_cpufreq_freq_up;
+		rte_power_freq_down = power_cppc_cpufreq_freq_down;
+		rte_power_freq_min = power_cppc_cpufreq_freq_min;
+		rte_power_freq_max = power_cppc_cpufreq_freq_max;
+		rte_power_turbo_status = power_cppc_turbo_status;
+		rte_power_freq_enable_turbo = power_cppc_enable_turbo;
+		rte_power_freq_disable_turbo = power_cppc_disable_turbo;
+		rte_power_get_capabilities = power_cppc_get_capabilities;
 	} else {
 		RTE_LOG(ERR, POWER, "Invalid Power Management Environment(%d) set\n",
 				env);
@@ -153,6 +168,8 @@ rte_power_init(unsigned int lcore_id)
 		return power_kvm_vm_init(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_init(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_init(lcore_id);
 	default:
 		RTE_LOG(INFO, POWER, "Env isn't set yet!\n");
 	}
@@ -172,6 +189,13 @@ rte_power_init(unsigned int lcore_id)
 		goto out;
 	}
 
+	RTE_LOG(INFO, POWER, "Attempting to initialise CPPC power management...\n");
+	ret = power_cppc_cpufreq_init(lcore_id);
+	if (ret == 0) {
+		rte_power_set_env(PM_ENV_CPPC_CPUFREQ);
+		goto out;
+	}
+
 	RTE_LOG(INFO, POWER, "Attempting to initialise VM power management...\n");
 	ret = power_kvm_vm_init(lcore_id);
 	if (ret == 0) {
@@ -194,6 +218,8 @@ rte_power_exit(unsigned int lcore_id)
 		return power_kvm_vm_exit(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_exit(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_exit(lcore_id);
 	default:
 		RTE_LOG(ERR, POWER, "Environment has not been set, unable to exit gracefully\n");
 
diff --git a/lib/power/rte_power.h b/lib/power/rte_power.h
index c8086bf6b..c5759afa3 100644
--- a/lib/power/rte_power.h
+++ b/lib/power/rte_power.h
@@ -22,7 +22,7 @@ extern "C" {
 
 /* Power Management Environment State */
 enum power_management_env {PM_ENV_NOT_SET, PM_ENV_ACPI_CPUFREQ, PM_ENV_KVM_VM,
-		PM_ENV_PSTATE_CPUFREQ};
+		PM_ENV_PSTATE_CPUFREQ, PM_ENV_CPPC_CPUFREQ};
 
 /**
  * @warning
-- 
2.20.1

Re: [dpdk-dev] [PATCH v4 1/2] power: add support for cppc cpufreq

[David Hunt]

Hi Richael

On 23/6/2021 4:55 AM, Richael Zhuang wrote:
> Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> supported, which are both not available on arm64 platforms. Add
> support for cppc_cpufreq driver which works on most arm64 platforms.
>
> Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> ---
>   app/test/test_power.c          |   3 +-
>   app/test/test_power_cpufreq.c  |   3 +-
>   lib/power/meson.build          |   1 +
>   lib/power/power_cppc_cpufreq.c | 632 +++++++++++++++++++++++++++++++++
>   lib/power/power_cppc_cpufreq.h | 229 ++++++++++++
>   lib/power/rte_power.c          |  26 ++
>   lib/power/rte_power.h          |   2 +-
>   7 files changed, 893 insertions(+), 3 deletions(-)
>   create mode 100644 lib/power/power_cppc_cpufreq.c
>   create mode 100644 lib/power/power_cppc_cpufreq.h
>
> diff --git a/app/test/test_power.c b/app/test/test_power.c
> index da1d67c0a..b7b556134 100644
> --- a/app/test/test_power.c
> +++ b/app/test/test_power.c
> @@ -133,7 +133,8 @@ test_power(void)
>   	/* Perform tests for valid environments.*/
>   	const enum power_management_env envs[] = {PM_ENV_ACPI_CPUFREQ,
>   			PM_ENV_KVM_VM,
> -			PM_ENV_PSTATE_CPUFREQ};
> +			PM_ENV_PSTATE_CPUFREQ,
> +			PM_ENV_CPPC_CPUFREQ};
>   
>   	unsigned int i;
>   	for (i = 0; i < RTE_DIM(envs); ++i) {
> diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
> index 0c3adc5f3..8516df4ca 100644
> --- a/app/test/test_power_cpufreq.c
> +++ b/app/test/test_power_cpufreq.c
> @@ -496,7 +496,8 @@ test_power_cpufreq(void)
>   
>   	/* Test environment configuration */
>   	env = rte_power_get_env();
> -	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ)) {
> +	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ) &&
> +			(env != PM_ENV_CPPC_CPUFREQ)) {
>   		printf("Unexpectedly got an environment other than ACPI/PSTATE\n");
>   		goto fail_all;
>   	}
> diff --git a/lib/power/meson.build b/lib/power/meson.build
> index 74c5f3a29..4a5b07292 100644
> --- a/lib/power/meson.build
> +++ b/lib/power/meson.build
> @@ -21,6 +21,7 @@ sources = files(
>           'rte_power.c',
>           'rte_power_empty_poll.c',
>           'rte_power_pmd_mgmt.c',
> +	'power_cppc_cpufreq.c',
>   )
>   headers = files(
>           'rte_power.h',
> diff --git a/lib/power/power_cppc_cpufreq.c b/lib/power/power_cppc_cpufreq.c
> new file mode 100644
> index 000000000..fd4483e52
> --- /dev/null
> +++ b/lib/power/power_cppc_cpufreq.c
> @@ -0,0 +1,632 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2021 Arm Limited


Since this is a port of the acpi or p-state code for the cppc driver, I 
think you need to keep the Intel line in there, and then add in the Arm 
Line. Suggest adding in:

  * Copyright(c) 2010-2021 Intel Corporation


Also, (I know you're looking at this already), the FOPS* and FOPEN* 
macros need to be removed to bring in line with the latest (v6) patch 
for the cleanup work, a few people have been wanting to remove those 
macros for a while now :)

Once that's done, this patch looks good to me, although I can't give it 
a run as I don't have access to a suitable system. But visually, I've 
read through the patch and it seems fine to me.

Reviewed-by: David Hunt <david.hunt@intel.com>


--snip--

Re: [dpdk-dev] [PATCH v4 1/2] power: add support for cppc cpufreq

[Richael Zhuang]

> -----Original Message-----
> From: David Hunt <david.hunt@intel.com>
> Sent: Wednesday, June 23, 2021 10:10 PM
> To: Richael Zhuang <Richael.Zhuang@arm.com>; dev@dpdk.org
> Cc: yux.jiang@intel.com
> Subject: Re: [PATCH v4 1/2] power: add support for cppc cpufreq
> 
> Hi Richael
> 
> On 23/6/2021 4:55 AM, Richael Zhuang wrote:
> > Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> > supported, which are both not available on arm64 platforms. Add
> > support for cppc_cpufreq driver which works on most arm64 platforms.
> >
> > Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> > ---
> >   app/test/test_power.c          |   3 +-
> >   app/test/test_power_cpufreq.c  |   3 +-
> >   lib/power/meson.build          |   1 +
> >   lib/power/power_cppc_cpufreq.c | 632
> +++++++++++++++++++++++++++++++++
> >   lib/power/power_cppc_cpufreq.h | 229 ++++++++++++
> >   lib/power/rte_power.c          |  26 ++
> >   lib/power/rte_power.h          |   2 +-
> >   7 files changed, 893 insertions(+), 3 deletions(-)
> >   create mode 100644 lib/power/power_cppc_cpufreq.c
> >   create mode 100644 lib/power/power_cppc_cpufreq.h
> >
> > diff --git a/app/test/test_power.c b/app/test/test_power.c index
> > da1d67c0a..b7b556134 100644
> > --- a/app/test/test_power.c
> > +++ b/app/test/test_power.c
> > @@ -133,7 +133,8 @@ test_power(void)
> >   	/* Perform tests for valid environments.*/
> >   	const enum power_management_env envs[] =
> {PM_ENV_ACPI_CPUFREQ,
> >   			PM_ENV_KVM_VM,
> > -			PM_ENV_PSTATE_CPUFREQ};
> > +			PM_ENV_PSTATE_CPUFREQ,
> > +			PM_ENV_CPPC_CPUFREQ};
> >
> >   	unsigned int i;
> >   	for (i = 0; i < RTE_DIM(envs); ++i) { diff --git
> > a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c index
> > 0c3adc5f3..8516df4ca 100644
> > --- a/app/test/test_power_cpufreq.c
> > +++ b/app/test/test_power_cpufreq.c
> > @@ -496,7 +496,8 @@ test_power_cpufreq(void)
> >
> >   	/* Test environment configuration */
> >   	env = rte_power_get_env();
> > -	if ((env != PM_ENV_ACPI_CPUFREQ) && (env !=
> PM_ENV_PSTATE_CPUFREQ)) {
> > +	if ((env != PM_ENV_ACPI_CPUFREQ) && (env !=
> PM_ENV_PSTATE_CPUFREQ) &&
> > +			(env != PM_ENV_CPPC_CPUFREQ)) {
> >   		printf("Unexpectedly got an environment other than
> ACPI/PSTATE\n");
> >   		goto fail_all;
> >   	}
> > diff --git a/lib/power/meson.build b/lib/power/meson.build index
> > 74c5f3a29..4a5b07292 100644
> > --- a/lib/power/meson.build
> > +++ b/lib/power/meson.build
> > @@ -21,6 +21,7 @@ sources = files(
> >           'rte_power.c',
> >           'rte_power_empty_poll.c',
> >           'rte_power_pmd_mgmt.c',
> > +	'power_cppc_cpufreq.c',
> >   )
> >   headers = files(
> >           'rte_power.h',
> > diff --git a/lib/power/power_cppc_cpufreq.c
> > b/lib/power/power_cppc_cpufreq.c new file mode 100644 index
> > 000000000..fd4483e52
> > --- /dev/null
> > +++ b/lib/power/power_cppc_cpufreq.c
> > @@ -0,0 +1,632 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2021 Arm Limited
> 
> 
> Since this is a port of the acpi or p-state code for the cppc driver, I think you
> need to keep the Intel line in there, and then add in the Arm Line. Suggest
> adding in:
> 
>   * Copyright(c) 2010-2021 Intel Corporation
> 
> 
> Also, (I know you're looking at this already), the FOPS* and FOPEN* macros
> need to be removed to bring in line with the latest (v6) patch for the cleanup
> work, a few people have been wanting to remove those macros for a while
> now :)
> 
Hi David,
Thanks! I will submit a new version soon.

> Once that's done, this patch looks good to me, although I can't give it a run as
> I don't have access to a suitable system. But visually, I've read through the
> patch and it seems fine to me.
> 
> Reviewed-by: David Hunt <david.hunt@intel.com>
> 
> 
> --snip--

[dpdk-dev] [PATCH v5 0/2] power: add support for cppc cpufreq driver

[Richael Zhuang]

v5:
rebase on Anatoly's patch: http://dpdk.org/patch/94763

Richael Zhuang (2):
  power: add support for cppc cpufreq
  test/power: round cpuinfo cur freq only when using CPPC cpufreq

 app/test/test_power.c          |   3 +-
 app/test/test_power_cpufreq.c  |  26 +-
 lib/power/meson.build          |   1 +
 lib/power/power_cppc_cpufreq.c | 681 +++++++++++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h | 230 +++++++++++
 lib/power/rte_power.c          |  26 ++
 lib/power/rte_power.h          |   2 +-
 7 files changed, 957 insertions(+), 12 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

-- 
2.20.1

[dpdk-dev] [PATCH v5 1/2] power: add support for cppc cpufreq

[Richael Zhuang]

Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
supported, which are both not available on arm64 platforms. Add
support for cppc_cpufreq driver which works on most arm64 platforms.

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
---
 app/test/test_power.c          |   3 +-
 app/test/test_power_cpufreq.c  |   3 +-
 lib/power/meson.build          |   1 +
 lib/power/power_cppc_cpufreq.c | 681 +++++++++++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h | 230 +++++++++++
 lib/power/rte_power.c          |  26 ++
 lib/power/rte_power.h          |   2 +-
 7 files changed, 943 insertions(+), 3 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

diff --git a/app/test/test_power.c b/app/test/test_power.c
index da1d67c0a..b7b556134 100644
--- a/app/test/test_power.c
+++ b/app/test/test_power.c
@@ -133,7 +133,8 @@ test_power(void)
 	/* Perform tests for valid environments.*/
 	const enum power_management_env envs[] = {PM_ENV_ACPI_CPUFREQ,
 			PM_ENV_KVM_VM,
-			PM_ENV_PSTATE_CPUFREQ};
+			PM_ENV_PSTATE_CPUFREQ,
+			PM_ENV_CPPC_CPUFREQ};
 
 	unsigned int i;
 	for (i = 0; i < RTE_DIM(envs); ++i) {
diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index 0c3adc5f3..8516df4ca 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -496,7 +496,8 @@ test_power_cpufreq(void)
 
 	/* Test environment configuration */
 	env = rte_power_get_env();
-	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ)) {
+	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ) &&
+			(env != PM_ENV_CPPC_CPUFREQ)) {
 		printf("Unexpectedly got an environment other than ACPI/PSTATE\n");
 		goto fail_all;
 	}
diff --git a/lib/power/meson.build b/lib/power/meson.build
index 74c5f3a29..4a5b07292 100644
--- a/lib/power/meson.build
+++ b/lib/power/meson.build
@@ -21,6 +21,7 @@ sources = files(
         'rte_power.c',
         'rte_power_empty_poll.c',
         'rte_power_pmd_mgmt.c',
+	'power_cppc_cpufreq.c',
 )
 headers = files(
         'rte_power.h',
diff --git a/lib/power/power_cppc_cpufreq.c b/lib/power/power_cppc_cpufreq.c
new file mode 100644
index 000000000..ea37e149a
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.c
@@ -0,0 +1,681 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2021 Intel Corporation
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#include <rte_memcpy.h>
+#include <rte_memory.h>
+
+#include "power_cppc_cpufreq.h"
+#include "power_common.h"
+
+/* macros used for rounding frequency to nearest 100000 */
+#define FREQ_ROUNDING_DELTA 50000
+#define ROUND_FREQ_TO_N_100000 100000
+
+/* the unit of highest_perf and nominal_perf differs on different arm platforms.
+ * For highest_perf, it maybe 300 or 3000000, both means 3.0GHz.
+ */
+#define UNIT_DIFF 10000
+
+#define POWER_CONVERT_TO_DECIMAL 10
+
+#define POWER_GOVERNOR_USERSPACE "userspace"
+#define POWER_SYSFILE_SETSPEED   \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_setspeed"
+#define POWER_SYSFILE_SCALING_MAX_FREQ \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
+#define POWER_SYSFILE_SCALING_MIN_FREQ  \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
+#define POWER_SYSFILE_HIGHEST_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/highest_perf"
+#define POWER_SYSFILE_NOMINAL_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/nominal_perf"
+#define POWER_SYSFILE_SYS_MAX \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
+
+#define POWER_CPPC_DRIVER "cppc-cpufreq"
+#define BUS_FREQ     100000
+
+enum power_state {
+	POWER_IDLE = 0,
+	POWER_ONGOING,
+	POWER_USED,
+	POWER_UNKNOWN
+};
+
+/**
+ * Power info per lcore.
+ */
+struct cppc_power_info {
+	unsigned int lcore_id;                   /**< Logical core id */
+	uint32_t state;                      /**< Power in use state */
+	FILE *f;                             /**< FD of scaling_setspeed */
+	char governor_ori[32];               /**< Original governor name */
+	uint32_t curr_idx;                   /**< Freq index in freqs array */
+	uint32_t highest_perf;		     /**< system wide max freq */
+	uint32_t nominal_perf;		     /**< system wide nominal freq */
+	uint16_t turbo_available;            /**< Turbo Boost available */
+	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
+	uint32_t nb_freqs;                   /**< number of available freqs */
+	uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
+} __rte_cache_aligned;
+
+static struct cppc_power_info lcore_power_info[RTE_MAX_LCORE];
+
+/**
+ * It is to set specific freq for specific logical core, according to the index
+ * of supported frequencies.
+ */
+static int
+set_freq_internal(struct cppc_power_info *pi, uint32_t idx)
+{
+	if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
+				"should be less than %u\n", idx, pi->nb_freqs);
+		return -1;
+	}
+
+	/* Check if it is the same as current */
+	if (idx == pi->curr_idx)
+		return 0;
+
+	POWER_DEBUG_TRACE("Frequency[%u] %u to be set for lcore %u\n",
+			idx, pi->freqs[idx], pi->lcore_id);
+	if (fseek(pi->f, 0, SEEK_SET) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
+			"for setting frequency for lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	if (fprintf(pi->f, "%u", pi->freqs[idx]) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new frequency for "
+				"lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	fflush(pi->f);
+	pi->curr_idx = idx;
+
+	return 1;
+}
+
+/**
+ * It is to check the current scaling governor by reading sys file, and then
+ * set it into 'userspace' if it is not by writing the sys file. The original
+ * governor will be saved for rolling back.
+ */
+static int
+power_set_governor_userspace(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, POWER_GOVERNOR_USERSPACE,
+			pi->governor_ori, sizeof(pi->governor_ori));
+}
+
+static int
+power_check_turbo(struct cppc_power_info *pi)
+{
+	FILE *f_nom, *f_max, *f_cmax;
+	int ret = -1;
+	uint32_t nominal_perf = 0, highest_perf = 0, cpuinfo_max_freq = 0;
+
+	open_core_sysfs_file(POWER_SYSFILE_HIGHEST_PERF, pi->lcore_id, "r",
+			&f_max);
+	if (f_max == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_HIGHEST_PERF);
+		goto err;
+	}
+
+	open_core_sysfs_file(POWER_SYSFILE_NOMINAL_PERF, pi->lcore_id, "r",
+			&f_nom);
+	if (f_nom == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_NOMINAL_PERF);
+		goto err;
+	}
+
+	open_core_sysfs_file(POWER_SYSFILE_SYS_MAX, pi->lcore_id, "r",
+			&f_cmax);
+	if (f_cmax == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_SYS_MAX);
+		goto err;
+	}
+
+	ret = read_core_sysfs_u32(f_max, &highest_perf);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_HIGHEST_PERF);
+		goto err;
+	}
+
+	ret = read_core_sysfs_u32(f_nom, &nominal_perf);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_NOMINAL_PERF);
+		goto err;
+	}
+
+	ret = read_core_sysfs_u32(f_cmax, &cpuinfo_max_freq);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_SYS_MAX);
+		goto err;
+	}
+
+	pi->highest_perf = highest_perf;
+	pi->nominal_perf = nominal_perf;
+
+	if ((highest_perf > nominal_perf) && ((cpuinfo_max_freq == highest_perf) ||
+				cpuinfo_max_freq == highest_perf * UNIT_DIFF)) {
+		pi->turbo_available = 1;
+		pi->turbo_enable = 1;
+		ret = 0;
+		POWER_DEBUG_TRACE("Lcore %u can do Turbo Boost! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	} else {
+		pi->turbo_available = 0;
+		pi->turbo_enable = 0;
+		POWER_DEBUG_TRACE("Lcore %u Turbo not available! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	}
+
+err:
+	if (f_max != NULL)
+		fclose(f_max);
+	if (f_nom != NULL)
+		fclose(f_nom);
+	if (f_cmax != NULL)
+		fclose(f_cmax);
+
+	return ret;
+}
+
+/**
+ * It is to get the available frequencies of the specific lcore by reading the
+ * sys file.
+ */
+static int
+power_get_available_freqs(struct cppc_power_info *pi)
+{
+	FILE *f_min, *f_max;
+	int ret = -1;
+	uint32_t scaling_min_freq = 0, scaling_max_freq = 0, nominal_perf = 0;
+	uint32_t i, num_freqs = 0;
+
+	open_core_sysfs_file(POWER_SYSFILE_SCALING_MAX_FREQ, pi->lcore_id, "r",
+			&f_max);
+	if (f_max == NULL) {
+                RTE_LOG(ERR, POWER, "failed to open %s\n",
+                                POWER_SYSFILE_SCALING_MAX_FREQ);
+                goto out;
+        }
+
+	open_core_sysfs_file(POWER_SYSFILE_SCALING_MIN_FREQ, pi->lcore_id, "r",
+			&f_min);
+	if (f_min == NULL) {
+                RTE_LOG(ERR, POWER, "failed to open %s\n",
+                                POWER_SYSFILE_SCALING_MIN_FREQ);
+                goto out;
+        }
+
+	ret = read_core_sysfs_u32(f_max, &scaling_max_freq);
+	if (ret < 0) {
+                RTE_LOG(ERR, POWER, "Failed to read %s\n",
+                                POWER_SYSFILE_SCALING_MAX_FREQ);
+                goto out;
+        }
+
+	ret = read_core_sysfs_u32(f_min, &scaling_min_freq);
+	if (ret < 0) {
+                RTE_LOG(ERR, POWER, "Failed to read %s\n",
+                                POWER_SYSFILE_SCALING_MIN_FREQ);
+                goto out;
+        }
+
+	power_check_turbo(pi);
+
+	if (scaling_max_freq < scaling_min_freq)
+		goto out;
+
+	/* If turbo is available then there is one extra freq bucket
+	 * to store the sys max freq which value is scaling_max_freq
+	 */
+	nominal_perf = (pi->nominal_perf < UNIT_DIFF) ?
+			pi->nominal_perf * UNIT_DIFF : pi->nominal_perf;
+	num_freqs = (nominal_perf - scaling_min_freq) / BUS_FREQ + 1 +
+		pi->turbo_available;
+
+	/* Generate the freq bucket array. */
+	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
+		if ((i == 0) && pi->turbo_available)
+			pi->freqs[pi->nb_freqs++] = scaling_max_freq;
+		else
+			pi->freqs[pi->nb_freqs++] =
+			nominal_perf - (i - pi->turbo_available) * BUS_FREQ;
+	}
+
+	ret = 0;
+
+	POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
+			num_freqs, pi->lcore_id);
+
+out:
+	if (f_min != NULL)
+		fclose(f_min);
+	if (f_max != NULL)
+		fclose(f_max);
+
+	return ret;
+}
+
+/**
+ * It is to fopen the sys file for the future setting the lcore frequency.
+ */
+static int
+power_init_for_setting_freq(struct cppc_power_info *pi)
+{
+	FILE *f;
+	char buf[BUFSIZ];
+	uint32_t i, freq;
+	int ret;
+
+	open_core_sysfs_file(POWER_SYSFILE_SETSPEED, pi->lcore_id, "rw+",
+			&f);
+	if (f == NULL) {
+                RTE_LOG(ERR, POWER, "failed to open %s\n",
+                                POWER_SYSFILE_SETSPEED);
+                goto err;
+        }
+
+	ret = read_core_sysfs_s(f, buf, sizeof(buf));
+	if (ret < 0) {
+                RTE_LOG(ERR, POWER, "Failed to read %s\n",
+                                POWER_SYSFILE_SETSPEED);
+                goto err;
+        }
+
+	freq = strtoul(buf, NULL, POWER_CONVERT_TO_DECIMAL);
+
+	/* convert the frequency to nearest 100000 value
+	 * Ex: if freq=1396789 then freq_conv=1400000
+	 * Ex: if freq=800030 then freq_conv=800000
+	 */
+	unsigned int freq_conv = 0;
+	freq_conv = (freq + FREQ_ROUNDING_DELTA)
+				/ ROUND_FREQ_TO_N_100000;
+	freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
+
+	for (i = 0; i < pi->nb_freqs; i++) {
+		if (freq_conv == pi->freqs[i]) {
+			pi->curr_idx = i;
+			pi->f = f;
+			return 0;
+		}
+	}
+
+err:
+	if (f != NULL)
+		fclose(f);
+
+	return -1;
+}
+
+int
+power_cppc_cpufreq_check_supported(void)
+{
+	return cpufreq_check_scaling_driver(POWER_CPPC_DRIVER);
+}
+
+int
+power_cppc_cpufreq_init(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_IDLE;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"in use\n", lcore_id);
+		return -1;
+	}
+
+	pi->lcore_id = lcore_id;
+	/* Check and set the governor */
+	if (power_set_governor_userspace(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
+				"userspace\n", lcore_id);
+		goto fail;
+	}
+
+	/* Get the available frequencies */
+	if (power_get_available_freqs(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Init for setting lcore frequency */
+	if (power_init_for_setting_freq(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Set freq to max by default */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
+				"to max\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
+			"power management\n", lcore_id);
+
+	__atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	return -1;
+}
+
+/**
+ * It is to check the governor and then set the original governor back if
+ * needed by writing the sys file.
+ */
+static int
+power_set_governor_original(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, pi->governor_ori, NULL, 0);
+}
+
+int
+power_cppc_cpufreq_exit(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_USED;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"not used\n", lcore_id);
+		return -1;
+	}
+
+	/* Close FD of setting freq */
+	fclose(pi->f);
+	pi->f = NULL;
+
+	/* Set the governor back to the original */
+	if (power_set_governor_original(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
+				"to the original\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
+			"'userspace' mode and been set back to the "
+			"original\n", lcore_id);
+	__atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+
+	return -1;
+}
+
+uint32_t
+power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return 0;
+	}
+
+	if (freqs == NULL) {
+		RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
+		return 0;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (num < pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
+		return 0;
+	}
+	rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
+
+	return pi->nb_freqs;
+}
+
+uint32_t
+power_cppc_cpufreq_get_freq(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return RTE_POWER_INVALID_FREQ_INDEX;
+	}
+
+	return lcore_power_info[lcore_id].curr_idx;
+}
+
+int
+power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	return set_freq_internal(&(lcore_power_info[lcore_id]), index);
+}
+
+int
+power_cppc_cpufreq_freq_down(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx + 1 == pi->nb_freqs)
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx + 1);
+}
+
+int
+power_cppc_cpufreq_freq_up(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx == 0 || (pi->curr_idx == 1 &&
+		pi->turbo_available && !pi->turbo_enable))
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx - 1);
+}
+
+int
+power_cppc_cpufreq_freq_max(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	/* Frequencies in the array are from high to low. */
+	if (lcore_power_info[lcore_id].turbo_available) {
+		if (lcore_power_info[lcore_id].turbo_enable)
+			/* Set to Turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 0);
+		else
+			/* Set to max non-turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 1);
+	} else
+		return set_freq_internal(&lcore_power_info[lcore_id], 0);
+}
+
+int
+power_cppc_cpufreq_freq_min(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->nb_freqs - 1);
+}
+
+int
+power_cppc_turbo_status(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	return pi->turbo_enable;
+}
+
+int
+power_cppc_enable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	if (pi->turbo_available)
+		pi->turbo_enable = 1;
+	else {
+		pi->turbo_enable = 0;
+		RTE_LOG(ERR, POWER,
+			"Failed to enable turbo on lcore %u\n",
+			lcore_id);
+		return -1;
+	}
+
+	/* TODO: must set to max once enbling Turbo? Considering add condition:
+	 * if ((pi->turbo_available) && (pi->curr_idx <= 1))
+	 */
+	/* Max may have changed, so call to max function */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER,
+			"Failed to set frequency of lcore %u to max\n",
+			lcore_id);
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+power_cppc_disable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	pi->turbo_enable = 0;
+
+	if ((pi->turbo_available) && (pi->curr_idx <= 1)) {
+		/* Try to set freq to max by default coming out of turbo */
+		if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+			RTE_LOG(ERR, POWER,
+				"Failed to set frequency of lcore %u to max\n",
+				lcore_id);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+int
+power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+	if (caps == NULL) {
+		RTE_LOG(ERR, POWER, "Invalid argument\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	caps->capabilities = 0;
+	caps->turbo = !!(pi->turbo_available);
+
+	return 0;
+}
diff --git a/lib/power/power_cppc_cpufreq.h b/lib/power/power_cppc_cpufreq.h
new file mode 100644
index 000000000..3bfc5f59c
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.h
@@ -0,0 +1,230 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2021 Intel Corporation
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#ifndef _POWER_CPPC_CPUFREQ_H
+#define _POWER_CPPC_CPUFREQ_H
+
+/**
+ * @file
+ * RTE Power Management via userspace CPPC cpufreq
+ */
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_string_fns.h>
+#include "rte_power.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Check if CPPC power management is supported.
+ *
+ * @return
+ *   - 1 if supported
+ *   - 0 if unsupported
+ *   - -1 if error, with rte_errno indicating reason for error.
+ */
+int power_cppc_cpufreq_check_supported(void);
+
+/**
+ * Initialize power management for a specific lcore. It will check and set the
+ * governor to userspace for the lcore, get the available frequencies, and
+ * prepare to set new lcore frequency.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_init(unsigned int lcore_id);
+
+/**
+ * Exit power management on a specific lcore. It will set the governor to which
+ * is before initialized.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_exit(unsigned int lcore_id);
+
+/**
+ * Get the available frequencies of a specific lcore. The return value will be
+ * the minimal one of the total number of available frequencies and the number
+ * of buffer. The index of available frequencies used in other interfaces
+ * should be in the range of 0 to this return value.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param freqs
+ *  The buffer array to save the frequencies.
+ * @param num
+ *  The number of frequencies to get.
+ *
+ * @return
+ *  The number of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs,
+		uint32_t num);
+
+/**
+ * Return the current index of available frequencies of a specific lcore. It
+ * will return 'RTE_POWER_INVALID_FREQ_INDEX = (~0)' if error.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  The current index of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_get_freq(unsigned int lcore_id);
+
+/**
+ * Set the new frequency for a specific lcore by indicating the index of
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param index
+ *  The index of available frequencies.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index);
+
+/**
+ * Scale up the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_up(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_down(unsigned int lcore_id);
+
+/**
+ * Scale up the frequency of a specific lcore to the highest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_max(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore to the lowest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_min(unsigned int lcore_id);
+
+/**
+ * Get the turbo status of a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 Turbo Boost is enabled on this lcore.
+ *  - 0 Turbo Boost is disabled on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_turbo_status(unsigned int lcore_id);
+
+/**
+ * Enable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost is enabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_enable_turbo(unsigned int lcore_id);
+
+/**
+ * Disable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost disabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_disable_turbo(unsigned int lcore_id);
+
+/**
+ * Returns power capabilities for a specific lcore.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param caps
+ *  pointer to rte_power_core_capabilities object.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/power/rte_power.c b/lib/power/rte_power.c
index 98eaba915..3d51ff8be 100644
--- a/lib/power/rte_power.c
+++ b/lib/power/rte_power.c
@@ -10,6 +10,7 @@
 #include "power_kvm_vm.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
+#include "power_cppc_cpufreq.h"
 
 enum power_management_env global_default_env = PM_ENV_NOT_SET;
 
@@ -54,6 +55,8 @@ rte_power_check_env_supported(enum power_management_env env)
 		return power_pstate_cpufreq_check_supported();
 	case PM_ENV_KVM_VM:
 		return power_kvm_vm_check_supported();
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_check_supported();
 	default:
 		rte_errno = EINVAL;
 		return -1;
@@ -110,6 +113,18 @@ rte_power_set_env(enum power_management_env env)
 		rte_power_freq_disable_turbo = power_pstate_disable_turbo;
 		rte_power_get_capabilities = power_pstate_get_capabilities;
 
+	} else if (env == PM_ENV_CPPC_CPUFREQ) {
+		rte_power_freqs = power_cppc_cpufreq_freqs;
+		rte_power_get_freq = power_cppc_cpufreq_get_freq;
+		rte_power_set_freq = power_cppc_cpufreq_set_freq;
+		rte_power_freq_up = power_cppc_cpufreq_freq_up;
+		rte_power_freq_down = power_cppc_cpufreq_freq_down;
+		rte_power_freq_min = power_cppc_cpufreq_freq_min;
+		rte_power_freq_max = power_cppc_cpufreq_freq_max;
+		rte_power_turbo_status = power_cppc_turbo_status;
+		rte_power_freq_enable_turbo = power_cppc_enable_turbo;
+		rte_power_freq_disable_turbo = power_cppc_disable_turbo;
+		rte_power_get_capabilities = power_cppc_get_capabilities;
 	} else {
 		RTE_LOG(ERR, POWER, "Invalid Power Management Environment(%d) set\n",
 				env);
@@ -153,6 +168,8 @@ rte_power_init(unsigned int lcore_id)
 		return power_kvm_vm_init(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_init(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_init(lcore_id);
 	default:
 		RTE_LOG(INFO, POWER, "Env isn't set yet!\n");
 	}
@@ -172,6 +189,13 @@ rte_power_init(unsigned int lcore_id)
 		goto out;
 	}
 
+	RTE_LOG(INFO, POWER, "Attempting to initialise CPPC power management...\n");
+	ret = power_cppc_cpufreq_init(lcore_id);
+	if (ret == 0) {
+		rte_power_set_env(PM_ENV_CPPC_CPUFREQ);
+		goto out;
+	}
+
 	RTE_LOG(INFO, POWER, "Attempting to initialise VM power management...\n");
 	ret = power_kvm_vm_init(lcore_id);
 	if (ret == 0) {
@@ -194,6 +218,8 @@ rte_power_exit(unsigned int lcore_id)
 		return power_kvm_vm_exit(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_exit(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_exit(lcore_id);
 	default:
 		RTE_LOG(ERR, POWER, "Environment has not been set, unable to exit gracefully\n");
 
diff --git a/lib/power/rte_power.h b/lib/power/rte_power.h
index c8086bf6b..c5759afa3 100644
--- a/lib/power/rte_power.h
+++ b/lib/power/rte_power.h
@@ -22,7 +22,7 @@ extern "C" {
 
 /* Power Management Environment State */
 enum power_management_env {PM_ENV_NOT_SET, PM_ENV_ACPI_CPUFREQ, PM_ENV_KVM_VM,
-		PM_ENV_PSTATE_CPUFREQ};
+		PM_ENV_PSTATE_CPUFREQ, PM_ENV_CPPC_CPUFREQ};
 
 /**
  * @warning
-- 
2.20.1

Re: [dpdk-dev] [PATCH v5 1/2] power: add support for cppc cpufreq

[David Hunt]

Hi Richael,

On 25/6/2021 3:02 AM, Richael Zhuang wrote:
> Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> supported, which are both not available on arm64 platforms. Add
> support for cppc_cpufreq driver which works on most arm64 platforms.
>
> Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> ---


Hi Richael,

The patch set builds fine when built on top of the refactor patch-set at 
http://patches.dpdk.org/project/dpdk/list/?series=17453

However, checkpatch finds several errors and warnings. Could we get a 
new version of the patch with these issues fixed?

Otherwise both patches in the set look good to me.

Rgds,

Dave.

Re: [dpdk-dev] [PATCH v5 1/2] power: add support for cppc cpufreq

[Richael Zhuang]

> -----Original Message-----
> From: David Hunt <david.hunt@intel.com>
> Sent: Wednesday, July 7, 2021 9:47 PM
> To: Richael Zhuang <Richael.Zhuang@arm.com>; dev@dpdk.org
> Cc: yux.jiang@intel.com
> Subject: Re: [PATCH v5 1/2] power: add support for cppc cpufreq
> 
> Hi Richael,
> 
> On 25/6/2021 3:02 AM, Richael Zhuang wrote:
> > Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> > supported, which are both not available on arm64 platforms. Add
> > support for cppc_cpufreq driver which works on most arm64 platforms.
> >
> > Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> > ---
> 
> 
> Hi Richael,
> 
> The patch set builds fine when built on top of the refactor patch-set at
> http://patches.dpdk.org/project/dpdk/list/?series=17453
> 
> However, checkpatch finds several errors and warnings. Could we get a new
> version of the patch with these issues fixed?
> 
> Otherwise both patches in the set look good to me.
> 
> Rgds,
> 
> Dave.
> 
Thanks! I will fix them in version6.

Rgds,
Richael
> 
> 

[dpdk-dev] [PATCH v6 0/2] power: add support for cppc cpufreq driver

[Richael Zhuang]

v6:
fix some checkpatch errors and warnings
rebase on Anatoly's patch: http://dpdk.org/patch/94763

Richael Zhuang (2):
  power: add support for cppc cpufreq
  test/power: round cpuinfo cur freq only in CPPC cpufreq

 app/test/test_power.c          |   3 +-
 app/test/test_power_cpufreq.c  |  26 +-
 lib/power/meson.build          |   1 +
 lib/power/power_cppc_cpufreq.c | 681 +++++++++++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h | 230 +++++++++++
 lib/power/rte_power.c          |  26 ++
 lib/power/rte_power.h          |   2 +-
 7 files changed, 957 insertions(+), 12 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

-- 
2.20.1

[dpdk-dev] [PATCH v6 1/2] power: add support for cppc cpufreq

[Richael Zhuang]

Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
supported, which are both not available on arm64 platforms. Add
support for cppc_cpufreq driver which works on most arm64 platforms.

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
---
 app/test/test_power.c          |   3 +-
 app/test/test_power_cpufreq.c  |   3 +-
 lib/power/meson.build          |   1 +
 lib/power/power_cppc_cpufreq.c | 681 +++++++++++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h | 230 +++++++++++
 lib/power/rte_power.c          |  26 ++
 lib/power/rte_power.h          |   2 +-
 7 files changed, 943 insertions(+), 3 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

diff --git a/app/test/test_power.c b/app/test/test_power.c
index da1d67c0a..b7b556134 100644
--- a/app/test/test_power.c
+++ b/app/test/test_power.c
@@ -133,7 +133,8 @@ test_power(void)
 	/* Perform tests for valid environments.*/
 	const enum power_management_env envs[] = {PM_ENV_ACPI_CPUFREQ,
 			PM_ENV_KVM_VM,
-			PM_ENV_PSTATE_CPUFREQ};
+			PM_ENV_PSTATE_CPUFREQ,
+			PM_ENV_CPPC_CPUFREQ};
 
 	unsigned int i;
 	for (i = 0; i < RTE_DIM(envs); ++i) {
diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index 0c3adc5f3..8516df4ca 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -496,7 +496,8 @@ test_power_cpufreq(void)
 
 	/* Test environment configuration */
 	env = rte_power_get_env();
-	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ)) {
+	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ) &&
+			(env != PM_ENV_CPPC_CPUFREQ)) {
 		printf("Unexpectedly got an environment other than ACPI/PSTATE\n");
 		goto fail_all;
 	}
diff --git a/lib/power/meson.build b/lib/power/meson.build
index 74c5f3a29..4a5b07292 100644
--- a/lib/power/meson.build
+++ b/lib/power/meson.build
@@ -21,6 +21,7 @@ sources = files(
         'rte_power.c',
         'rte_power_empty_poll.c',
         'rte_power_pmd_mgmt.c',
+	'power_cppc_cpufreq.c',
 )
 headers = files(
         'rte_power.h',
diff --git a/lib/power/power_cppc_cpufreq.c b/lib/power/power_cppc_cpufreq.c
new file mode 100644
index 000000000..3411d5d3d
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.c
@@ -0,0 +1,681 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2021 Intel Corporation
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#include <rte_memcpy.h>
+#include <rte_memory.h>
+
+#include "power_cppc_cpufreq.h"
+#include "power_common.h"
+
+/* macros used for rounding frequency to nearest 100000 */
+#define FREQ_ROUNDING_DELTA 50000
+#define ROUND_FREQ_TO_N_100000 100000
+
+/* the unit of highest_perf and nominal_perf differs on different arm platforms.
+ * For highest_perf, it maybe 300 or 3000000, both means 3.0GHz.
+ */
+#define UNIT_DIFF 10000
+
+#define POWER_CONVERT_TO_DECIMAL 10
+
+#define POWER_GOVERNOR_USERSPACE "userspace"
+#define POWER_SYSFILE_SETSPEED   \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_setspeed"
+#define POWER_SYSFILE_SCALING_MAX_FREQ \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
+#define POWER_SYSFILE_SCALING_MIN_FREQ  \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
+#define POWER_SYSFILE_HIGHEST_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/highest_perf"
+#define POWER_SYSFILE_NOMINAL_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/nominal_perf"
+#define POWER_SYSFILE_SYS_MAX \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
+
+#define POWER_CPPC_DRIVER "cppc-cpufreq"
+#define BUS_FREQ     100000
+
+enum power_state {
+	POWER_IDLE = 0,
+	POWER_ONGOING,
+	POWER_USED,
+	POWER_UNKNOWN
+};
+
+/**
+ * Power info per lcore.
+ */
+struct cppc_power_info {
+	unsigned int lcore_id;                   /**< Logical core id */
+	uint32_t state;                      /**< Power in use state */
+	FILE *f;                             /**< FD of scaling_setspeed */
+	char governor_ori[32];               /**< Original governor name */
+	uint32_t curr_idx;                   /**< Freq index in freqs array */
+	uint32_t highest_perf;		     /**< system wide max freq */
+	uint32_t nominal_perf;		     /**< system wide nominal freq */
+	uint16_t turbo_available;            /**< Turbo Boost available */
+	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
+	uint32_t nb_freqs;                   /**< number of available freqs */
+	uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
+} __rte_cache_aligned;
+
+static struct cppc_power_info lcore_power_info[RTE_MAX_LCORE];
+
+/**
+ * It is to set specific freq for specific logical core, according to the index
+ * of supported frequencies.
+ */
+static int
+set_freq_internal(struct cppc_power_info *pi, uint32_t idx)
+{
+	if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
+				"should be less than %u\n", idx, pi->nb_freqs);
+		return -1;
+	}
+
+	/* Check if it is the same as current */
+	if (idx == pi->curr_idx)
+		return 0;
+
+	POWER_DEBUG_TRACE("Frequency[%u] %u to be set for lcore %u\n",
+			idx, pi->freqs[idx], pi->lcore_id);
+	if (fseek(pi->f, 0, SEEK_SET) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
+			"for setting frequency for lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	if (fprintf(pi->f, "%u", pi->freqs[idx]) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new frequency for "
+				"lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	fflush(pi->f);
+	pi->curr_idx = idx;
+
+	return 1;
+}
+
+/**
+ * It is to check the current scaling governor by reading sys file, and then
+ * set it into 'userspace' if it is not by writing the sys file. The original
+ * governor will be saved for rolling back.
+ */
+static int
+power_set_governor_userspace(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, POWER_GOVERNOR_USERSPACE,
+			pi->governor_ori, sizeof(pi->governor_ori));
+}
+
+static int
+power_check_turbo(struct cppc_power_info *pi)
+{
+	FILE *f_nom, *f_max, *f_cmax;
+	int ret = -1;
+	uint32_t nominal_perf = 0, highest_perf = 0, cpuinfo_max_freq = 0;
+
+	open_core_sysfs_file(POWER_SYSFILE_HIGHEST_PERF, pi->lcore_id, "r",
+			&f_max);
+	if (f_max == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_HIGHEST_PERF);
+		goto err;
+	}
+
+	open_core_sysfs_file(POWER_SYSFILE_NOMINAL_PERF, pi->lcore_id, "r",
+			&f_nom);
+	if (f_nom == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_NOMINAL_PERF);
+		goto err;
+	}
+
+	open_core_sysfs_file(POWER_SYSFILE_SYS_MAX, pi->lcore_id, "r",
+			&f_cmax);
+	if (f_cmax == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_SYS_MAX);
+		goto err;
+	}
+
+	ret = read_core_sysfs_u32(f_max, &highest_perf);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_HIGHEST_PERF);
+		goto err;
+	}
+
+	ret = read_core_sysfs_u32(f_nom, &nominal_perf);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_NOMINAL_PERF);
+		goto err;
+	}
+
+	ret = read_core_sysfs_u32(f_cmax, &cpuinfo_max_freq);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_SYS_MAX);
+		goto err;
+	}
+
+	pi->highest_perf = highest_perf;
+	pi->nominal_perf = nominal_perf;
+
+	if ((highest_perf > nominal_perf) && ((cpuinfo_max_freq == highest_perf)
+			|| cpuinfo_max_freq == highest_perf * UNIT_DIFF)) {
+		pi->turbo_available = 1;
+		pi->turbo_enable = 1;
+		ret = 0;
+		POWER_DEBUG_TRACE("Lcore %u can do Turbo Boost! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	} else {
+		pi->turbo_available = 0;
+		pi->turbo_enable = 0;
+		POWER_DEBUG_TRACE("Lcore %u Turbo not available! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	}
+
+err:
+	if (f_max != NULL)
+		fclose(f_max);
+	if (f_nom != NULL)
+		fclose(f_nom);
+	if (f_cmax != NULL)
+		fclose(f_cmax);
+
+	return ret;
+}
+
+/**
+ * It is to get the available frequencies of the specific lcore by reading the
+ * sys file.
+ */
+static int
+power_get_available_freqs(struct cppc_power_info *pi)
+{
+	FILE *f_min, *f_max;
+	int ret = -1;
+	uint32_t scaling_min_freq = 0, scaling_max_freq = 0, nominal_perf = 0;
+	uint32_t i, num_freqs = 0;
+
+	open_core_sysfs_file(POWER_SYSFILE_SCALING_MAX_FREQ, pi->lcore_id, "r",
+			&f_max);
+	if (f_max == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_SCALING_MAX_FREQ);
+		goto out;
+	}
+
+	open_core_sysfs_file(POWER_SYSFILE_SCALING_MIN_FREQ, pi->lcore_id, "r",
+			&f_min);
+	if (f_min == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_SCALING_MIN_FREQ);
+		goto out;
+	}
+
+	ret = read_core_sysfs_u32(f_max, &scaling_max_freq);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_SCALING_MAX_FREQ);
+		goto out;
+	}
+
+	ret = read_core_sysfs_u32(f_min, &scaling_min_freq);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_SCALING_MIN_FREQ);
+		goto out;
+	}
+
+	power_check_turbo(pi);
+
+	if (scaling_max_freq < scaling_min_freq)
+		goto out;
+
+	/* If turbo is available then there is one extra freq bucket
+	 * to store the sys max freq which value is scaling_max_freq
+	 */
+	nominal_perf = (pi->nominal_perf < UNIT_DIFF) ?
+			pi->nominal_perf * UNIT_DIFF : pi->nominal_perf;
+	num_freqs = (nominal_perf - scaling_min_freq) / BUS_FREQ + 1 +
+		pi->turbo_available;
+
+	/* Generate the freq bucket array. */
+	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
+		if ((i == 0) && pi->turbo_available)
+			pi->freqs[pi->nb_freqs++] = scaling_max_freq;
+		else
+			pi->freqs[pi->nb_freqs++] =
+			nominal_perf - (i - pi->turbo_available) * BUS_FREQ;
+	}
+
+	ret = 0;
+
+	POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
+			num_freqs, pi->lcore_id);
+
+out:
+	if (f_min != NULL)
+		fclose(f_min);
+	if (f_max != NULL)
+		fclose(f_max);
+
+	return ret;
+}
+
+/**
+ * It is to fopen the sys file for the future setting the lcore frequency.
+ */
+static int
+power_init_for_setting_freq(struct cppc_power_info *pi)
+{
+	FILE *f;
+	char buf[BUFSIZ];
+	uint32_t i, freq;
+	int ret;
+
+	open_core_sysfs_file(POWER_SYSFILE_SETSPEED, pi->lcore_id, "rw+",
+			&f);
+	if (f == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_SETSPEED);
+		goto err;
+	}
+
+	ret = read_core_sysfs_s(f, buf, sizeof(buf));
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_SETSPEED);
+		goto err;
+	}
+
+	freq = strtoul(buf, NULL, POWER_CONVERT_TO_DECIMAL);
+
+	/* convert the frequency to nearest 100000 value
+	 * Ex: if freq=1396789 then freq_conv=1400000
+	 * Ex: if freq=800030 then freq_conv=800000
+	 */
+	unsigned int freq_conv = 0;
+	freq_conv = (freq + FREQ_ROUNDING_DELTA)
+				/ ROUND_FREQ_TO_N_100000;
+	freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
+
+	for (i = 0; i < pi->nb_freqs; i++) {
+		if (freq_conv == pi->freqs[i]) {
+			pi->curr_idx = i;
+			pi->f = f;
+			return 0;
+		}
+	}
+
+err:
+	if (f != NULL)
+		fclose(f);
+
+	return -1;
+}
+
+int
+power_cppc_cpufreq_check_supported(void)
+{
+	return cpufreq_check_scaling_driver(POWER_CPPC_DRIVER);
+}
+
+int
+power_cppc_cpufreq_init(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_IDLE;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"in use\n", lcore_id);
+		return -1;
+	}
+
+	pi->lcore_id = lcore_id;
+	/* Check and set the governor */
+	if (power_set_governor_userspace(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
+				"userspace\n", lcore_id);
+		goto fail;
+	}
+
+	/* Get the available frequencies */
+	if (power_get_available_freqs(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Init for setting lcore frequency */
+	if (power_init_for_setting_freq(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Set freq to max by default */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
+				"to max\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
+			"power management\n", lcore_id);
+
+	__atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	return -1;
+}
+
+/**
+ * It is to check the governor and then set the original governor back if
+ * needed by writing the sys file.
+ */
+static int
+power_set_governor_original(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, pi->governor_ori, NULL, 0);
+}
+
+int
+power_cppc_cpufreq_exit(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_USED;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"not used\n", lcore_id);
+		return -1;
+	}
+
+	/* Close FD of setting freq */
+	fclose(pi->f);
+	pi->f = NULL;
+
+	/* Set the governor back to the original */
+	if (power_set_governor_original(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
+				"to the original\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
+			"'userspace' mode and been set back to the "
+			"original\n", lcore_id);
+	__atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+
+	return -1;
+}
+
+uint32_t
+power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return 0;
+	}
+
+	if (freqs == NULL) {
+		RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
+		return 0;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (num < pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
+		return 0;
+	}
+	rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
+
+	return pi->nb_freqs;
+}
+
+uint32_t
+power_cppc_cpufreq_get_freq(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return RTE_POWER_INVALID_FREQ_INDEX;
+	}
+
+	return lcore_power_info[lcore_id].curr_idx;
+}
+
+int
+power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	return set_freq_internal(&(lcore_power_info[lcore_id]), index);
+}
+
+int
+power_cppc_cpufreq_freq_down(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx + 1 == pi->nb_freqs)
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx + 1);
+}
+
+int
+power_cppc_cpufreq_freq_up(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx == 0 || (pi->curr_idx == 1 &&
+		pi->turbo_available && !pi->turbo_enable))
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx - 1);
+}
+
+int
+power_cppc_cpufreq_freq_max(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	/* Frequencies in the array are from high to low. */
+	if (lcore_power_info[lcore_id].turbo_available) {
+		if (lcore_power_info[lcore_id].turbo_enable)
+			/* Set to Turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 0);
+		else
+			/* Set to max non-turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 1);
+	} else
+		return set_freq_internal(&lcore_power_info[lcore_id], 0);
+}
+
+int
+power_cppc_cpufreq_freq_min(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->nb_freqs - 1);
+}
+
+int
+power_cppc_turbo_status(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	return pi->turbo_enable;
+}
+
+int
+power_cppc_enable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	if (pi->turbo_available)
+		pi->turbo_enable = 1;
+	else {
+		pi->turbo_enable = 0;
+		RTE_LOG(ERR, POWER,
+			"Failed to enable turbo on lcore %u\n",
+			lcore_id);
+		return -1;
+	}
+
+	/* TODO: must set to max once enbling Turbo? Considering add condition:
+	 * if ((pi->turbo_available) && (pi->curr_idx <= 1))
+	 */
+	/* Max may have changed, so call to max function */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER,
+			"Failed to set frequency of lcore %u to max\n",
+			lcore_id);
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+power_cppc_disable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	pi->turbo_enable = 0;
+
+	if ((pi->turbo_available) && (pi->curr_idx <= 1)) {
+		/* Try to set freq to max by default coming out of turbo */
+		if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+			RTE_LOG(ERR, POWER,
+				"Failed to set frequency of lcore %u to max\n",
+				lcore_id);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+int
+power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+	if (caps == NULL) {
+		RTE_LOG(ERR, POWER, "Invalid argument\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	caps->capabilities = 0;
+	caps->turbo = !!(pi->turbo_available);
+
+	return 0;
+}
diff --git a/lib/power/power_cppc_cpufreq.h b/lib/power/power_cppc_cpufreq.h
new file mode 100644
index 000000000..3bfc5f59c
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.h
@@ -0,0 +1,230 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2021 Intel Corporation
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#ifndef _POWER_CPPC_CPUFREQ_H
+#define _POWER_CPPC_CPUFREQ_H
+
+/**
+ * @file
+ * RTE Power Management via userspace CPPC cpufreq
+ */
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_string_fns.h>
+#include "rte_power.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Check if CPPC power management is supported.
+ *
+ * @return
+ *   - 1 if supported
+ *   - 0 if unsupported
+ *   - -1 if error, with rte_errno indicating reason for error.
+ */
+int power_cppc_cpufreq_check_supported(void);
+
+/**
+ * Initialize power management for a specific lcore. It will check and set the
+ * governor to userspace for the lcore, get the available frequencies, and
+ * prepare to set new lcore frequency.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_init(unsigned int lcore_id);
+
+/**
+ * Exit power management on a specific lcore. It will set the governor to which
+ * is before initialized.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_exit(unsigned int lcore_id);
+
+/**
+ * Get the available frequencies of a specific lcore. The return value will be
+ * the minimal one of the total number of available frequencies and the number
+ * of buffer. The index of available frequencies used in other interfaces
+ * should be in the range of 0 to this return value.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param freqs
+ *  The buffer array to save the frequencies.
+ * @param num
+ *  The number of frequencies to get.
+ *
+ * @return
+ *  The number of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs,
+		uint32_t num);
+
+/**
+ * Return the current index of available frequencies of a specific lcore. It
+ * will return 'RTE_POWER_INVALID_FREQ_INDEX = (~0)' if error.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  The current index of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_get_freq(unsigned int lcore_id);
+
+/**
+ * Set the new frequency for a specific lcore by indicating the index of
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param index
+ *  The index of available frequencies.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index);
+
+/**
+ * Scale up the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_up(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_down(unsigned int lcore_id);
+
+/**
+ * Scale up the frequency of a specific lcore to the highest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_max(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore to the lowest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_min(unsigned int lcore_id);
+
+/**
+ * Get the turbo status of a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 Turbo Boost is enabled on this lcore.
+ *  - 0 Turbo Boost is disabled on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_turbo_status(unsigned int lcore_id);
+
+/**
+ * Enable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost is enabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_enable_turbo(unsigned int lcore_id);
+
+/**
+ * Disable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost disabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_disable_turbo(unsigned int lcore_id);
+
+/**
+ * Returns power capabilities for a specific lcore.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param caps
+ *  pointer to rte_power_core_capabilities object.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/power/rte_power.c b/lib/power/rte_power.c
index 98eaba915..3d51ff8be 100644
--- a/lib/power/rte_power.c
+++ b/lib/power/rte_power.c
@@ -10,6 +10,7 @@
 #include "power_kvm_vm.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
+#include "power_cppc_cpufreq.h"
 
 enum power_management_env global_default_env = PM_ENV_NOT_SET;
 
@@ -54,6 +55,8 @@ rte_power_check_env_supported(enum power_management_env env)
 		return power_pstate_cpufreq_check_supported();
 	case PM_ENV_KVM_VM:
 		return power_kvm_vm_check_supported();
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_check_supported();
 	default:
 		rte_errno = EINVAL;
 		return -1;
@@ -110,6 +113,18 @@ rte_power_set_env(enum power_management_env env)
 		rte_power_freq_disable_turbo = power_pstate_disable_turbo;
 		rte_power_get_capabilities = power_pstate_get_capabilities;
 
+	} else if (env == PM_ENV_CPPC_CPUFREQ) {
+		rte_power_freqs = power_cppc_cpufreq_freqs;
+		rte_power_get_freq = power_cppc_cpufreq_get_freq;
+		rte_power_set_freq = power_cppc_cpufreq_set_freq;
+		rte_power_freq_up = power_cppc_cpufreq_freq_up;
+		rte_power_freq_down = power_cppc_cpufreq_freq_down;
+		rte_power_freq_min = power_cppc_cpufreq_freq_min;
+		rte_power_freq_max = power_cppc_cpufreq_freq_max;
+		rte_power_turbo_status = power_cppc_turbo_status;
+		rte_power_freq_enable_turbo = power_cppc_enable_turbo;
+		rte_power_freq_disable_turbo = power_cppc_disable_turbo;
+		rte_power_get_capabilities = power_cppc_get_capabilities;
 	} else {
 		RTE_LOG(ERR, POWER, "Invalid Power Management Environment(%d) set\n",
 				env);
@@ -153,6 +168,8 @@ rte_power_init(unsigned int lcore_id)
 		return power_kvm_vm_init(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_init(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_init(lcore_id);
 	default:
 		RTE_LOG(INFO, POWER, "Env isn't set yet!\n");
 	}
@@ -172,6 +189,13 @@ rte_power_init(unsigned int lcore_id)
 		goto out;
 	}
 
+	RTE_LOG(INFO, POWER, "Attempting to initialise CPPC power management...\n");
+	ret = power_cppc_cpufreq_init(lcore_id);
+	if (ret == 0) {
+		rte_power_set_env(PM_ENV_CPPC_CPUFREQ);
+		goto out;
+	}
+
 	RTE_LOG(INFO, POWER, "Attempting to initialise VM power management...\n");
 	ret = power_kvm_vm_init(lcore_id);
 	if (ret == 0) {
@@ -194,6 +218,8 @@ rte_power_exit(unsigned int lcore_id)
 		return power_kvm_vm_exit(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_exit(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_exit(lcore_id);
 	default:
 		RTE_LOG(ERR, POWER, "Environment has not been set, unable to exit gracefully\n");
 
diff --git a/lib/power/rte_power.h b/lib/power/rte_power.h
index c8086bf6b..c5759afa3 100644
--- a/lib/power/rte_power.h
+++ b/lib/power/rte_power.h
@@ -22,7 +22,7 @@ extern "C" {
 
 /* Power Management Environment State */
 enum power_management_env {PM_ENV_NOT_SET, PM_ENV_ACPI_CPUFREQ, PM_ENV_KVM_VM,
-		PM_ENV_PSTATE_CPUFREQ};
+		PM_ENV_PSTATE_CPUFREQ, PM_ENV_CPPC_CPUFREQ};
 
 /**
  * @warning
-- 
2.20.1

Re: [dpdk-dev] [PATCH v6 1/2] power: add support for cppc cpufreq

[David Hunt]

On 8/7/2021 3:34 AM, Richael Zhuang wrote:
> Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> supported, which are both not available on arm64 platforms. Add
> support for cppc_cpufreq driver which works on most arm64 platforms.
>
> Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> ---
>   app/test/test_power.c          |   3 +-
>   app/test/test_power_cpufreq.c  |   3 +-
>   lib/power/meson.build          |   1 +
>   lib/power/power_cppc_cpufreq.c | 681 +++++++++++++++++++++++++++++++++
>   lib/power/power_cppc_cpufreq.h | 230 +++++++++++
>   lib/power/rte_power.c          |  26 ++
>   lib/power/rte_power.h          |   2 +-
>   7 files changed, 943 insertions(+), 3 deletions(-)
>   create mode 100644 lib/power/power_cppc_cpufreq.c
>   create mode 100644 lib/power/power_cppc_cpufreq.h
>

--snip--

Hi Richael, thanks for the quick re-spin.

Although this shows a build error in patchwork, as we know, that's 
because of the depenedent patch set - 
http://patches.dpdk.org/project/dpdk/list/?series=17453 
<http://patches.dpdk.org/project/dpdk/list/?series=17453>

If I apply that dependency first, it then builds fine, and checkpatch is 
clean, although I can't give it a run, as I currently don't have access 
to a suitable machine. LGTM.

Acked-by: David Hunt <david.hunt@intel.com>

Re: [dpdk-dev] [PATCH v6 1/2] power: add support for cppc cpufreq

[David Marchand]

On Thu, Jul 8, 2021 at 4:35 AM Richael Zhuang <richael.zhuang@arm.com> wrote:
>
> Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> supported, which are both not available on arm64 platforms. Add
> support for cppc_cpufreq driver which works on most arm64 platforms.

Worth a release note update.
WDYT of:

diff --git a/doc/guides/rel_notes/release_21_08.rst
b/doc/guides/rel_notes/release_21_08.rst
index c92e016783..6fd9f0168a 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -103,6 +103,10 @@ New Features
   usecases. Configuration happens via standard rawdev enq/deq operations. See
   the :doc:`../rawdevs/cnxk_bphy` rawdev guide for more details on this driver.

+* **Added cppc_cpufreq support to Power Management library.**
+
+  Added support for cppc_cpufreq driver which works on most arm64 platforms.
+

 Removed Items
 -------------


>
> Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> ---
>  app/test/test_power.c          |   3 +-
>  app/test/test_power_cpufreq.c  |   3 +-
>  lib/power/meson.build          |   1 +
>  lib/power/power_cppc_cpufreq.c | 681 +++++++++++++++++++++++++++++++++
>  lib/power/power_cppc_cpufreq.h | 230 +++++++++++
>  lib/power/rte_power.c          |  26 ++
>  lib/power/rte_power.h          |   2 +-
>  7 files changed, 943 insertions(+), 3 deletions(-)
>  create mode 100644 lib/power/power_cppc_cpufreq.c
>  create mode 100644 lib/power/power_cppc_cpufreq.h
>
> diff --git a/app/test/test_power.c b/app/test/test_power.c
> index da1d67c0a..b7b556134 100644
> --- a/app/test/test_power.c
> +++ b/app/test/test_power.c
> @@ -133,7 +133,8 @@ test_power(void)
>         /* Perform tests for valid environments.*/
>         const enum power_management_env envs[] = {PM_ENV_ACPI_CPUFREQ,
>                         PM_ENV_KVM_VM,
> -                       PM_ENV_PSTATE_CPUFREQ};
> +                       PM_ENV_PSTATE_CPUFREQ,
> +                       PM_ENV_CPPC_CPUFREQ};
>
>         unsigned int i;
>         for (i = 0; i < RTE_DIM(envs); ++i) {
> diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
> index 0c3adc5f3..8516df4ca 100644
> --- a/app/test/test_power_cpufreq.c
> +++ b/app/test/test_power_cpufreq.c
> @@ -496,7 +496,8 @@ test_power_cpufreq(void)
>
>         /* Test environment configuration */
>         env = rte_power_get_env();
> -       if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ)) {
> +       if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ) &&
> +                       (env != PM_ENV_CPPC_CPUFREQ)) {
>                 printf("Unexpectedly got an environment other than ACPI/PSTATE\n");
>                 goto fail_all;
>         }
> diff --git a/lib/power/meson.build b/lib/power/meson.build
> index 74c5f3a29..4a5b07292 100644
> --- a/lib/power/meson.build
> +++ b/lib/power/meson.build
> @@ -21,6 +21,7 @@ sources = files(
>          'rte_power.c',
>          'rte_power_empty_poll.c',
>          'rte_power_pmd_mgmt.c',
> +       'power_cppc_cpufreq.c',

Wrong indent + worth sorting alphabetically.


>  )
>  headers = files(
>          'rte_power.h',

[snip]

> diff --git a/lib/power/power_cppc_cpufreq.h b/lib/power/power_cppc_cpufreq.h
> new file mode 100644
> index 000000000..3bfc5f59c
> --- /dev/null
> +++ b/lib/power/power_cppc_cpufreq.h
> @@ -0,0 +1,230 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2010-2021 Intel Corporation
> + * Copyright(c) 2021 Arm Limited
> + */
> +
> +#ifndef _POWER_CPPC_CPUFREQ_H
> +#define _POWER_CPPC_CPUFREQ_H
> +
> +/**
> + * @file
> + * RTE Power Management via userspace CPPC cpufreq
> + */
> +
> +#include <rte_common.h>
> +#include <rte_byteorder.h>
> +#include <rte_log.h>
> +#include <rte_string_fns.h>
> +#include "rte_power.h"
> +

[snip]

> +#endif

Nit: #endif /* _POWER_CPPC_CPUFREQ_H */


> diff --git a/lib/power/rte_power.c b/lib/power/rte_power.c
> index 98eaba915..3d51ff8be 100644
> --- a/lib/power/rte_power.c
> +++ b/lib/power/rte_power.c
> @@ -10,6 +10,7 @@
>  #include "power_kvm_vm.h"
>  #include "power_pstate_cpufreq.h"
>  #include "power_common.h"
> +#include "power_cppc_cpufreq.h"

Nit: this include can be sorted alphabetically with other driver headers above.


>
>  enum power_management_env global_default_env = PM_ENV_NOT_SET;
>


-- 
David Marchand

Re: [dpdk-dev] [PATCH v6 1/2] power: add support for cppc cpufreq

[David Marchand]

On Thu, Jul 8, 2021 at 3:30 PM David Marchand <david.marchand@redhat.com> wrote:
>
> On Thu, Jul 8, 2021 at 4:35 AM Richael Zhuang <richael.zhuang@arm.com> wrote:
> >
> > Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> > supported, which are both not available on arm64 platforms. Add
> > support for cppc_cpufreq driver which works on most arm64 platforms.
>
> Worth a release note update.
> WDYT of:
>
> diff --git a/doc/guides/rel_notes/release_21_08.rst
> b/doc/guides/rel_notes/release_21_08.rst
> index c92e016783..6fd9f0168a 100644
> --- a/doc/guides/rel_notes/release_21_08.rst
> +++ b/doc/guides/rel_notes/release_21_08.rst
> @@ -103,6 +103,10 @@ New Features
>    usecases. Configuration happens via standard rawdev enq/deq operations. See
>    the :doc:`../rawdevs/cnxk_bphy` rawdev guide for more details on this driver.
>
> +* **Added cppc_cpufreq support to Power Management library.**
> +
> +  Added support for cppc_cpufreq driver which works on most arm64 platforms.
> +
>
>  Removed Items
>  -------------
>
>

Please respin on current main branch as there was one small update on
lib/power internal API.
Thanks.


-- 
David Marchand

Re: [dpdk-dev] [PATCH v6 1/2] power: add support for cppc cpufreq

[Richael Zhuang]

Thanks so much, I will rework it soon.

Best Regards,
Richael
> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Thursday, July 8, 2021 9:30 PM
> To: Richael Zhuang <Richael.Zhuang@arm.com>
> Cc: dev <dev@dpdk.org>; Yu Jiang <yux.jiang@intel.com>; David Hunt
> <david.hunt@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v6 1/2] power: add support for cppc cpufreq
> 
> On Thu, Jul 8, 2021 at 4:35 AM Richael Zhuang <richael.zhuang@arm.com>
> wrote:
> >
> > Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> > supported, which are both not available on arm64 platforms. Add
> > support for cppc_cpufreq driver which works on most arm64 platforms.
> 
> Worth a release note update.
> WDYT of:
> 
> diff --git a/doc/guides/rel_notes/release_21_08.rst
> b/doc/guides/rel_notes/release_21_08.rst
> index c92e016783..6fd9f0168a 100644
> --- a/doc/guides/rel_notes/release_21_08.rst
> +++ b/doc/guides/rel_notes/release_21_08.rst
> @@ -103,6 +103,10 @@ New Features
>    usecases. Configuration happens via standard rawdev enq/deq operations.
> See
>    the :doc:`../rawdevs/cnxk_bphy` rawdev guide for more details on this
> driver.
> 
> +* **Added cppc_cpufreq support to Power Management library.**
> +
> +  Added support for cppc_cpufreq driver which works on most arm64
> platforms.
> +
> 
>  Removed Items
>  -------------
> 
> 
> >
> > Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> > ---
> >  app/test/test_power.c          |   3 +-
> >  app/test/test_power_cpufreq.c  |   3 +-
> >  lib/power/meson.build          |   1 +
> >  lib/power/power_cppc_cpufreq.c | 681
> > +++++++++++++++++++++++++++++++++  lib/power/power_cppc_cpufreq.h
> | 230 +++++++++++
> >  lib/power/rte_power.c          |  26 ++
> >  lib/power/rte_power.h          |   2 +-
> >  7 files changed, 943 insertions(+), 3 deletions(-)  create mode
> > 100644 lib/power/power_cppc_cpufreq.c  create mode 100644
> > lib/power/power_cppc_cpufreq.h
> >
> > diff --git a/app/test/test_power.c b/app/test/test_power.c index
> > da1d67c0a..b7b556134 100644
> > --- a/app/test/test_power.c
> > +++ b/app/test/test_power.c
> > @@ -133,7 +133,8 @@ test_power(void)
> >         /* Perform tests for valid environments.*/
> >         const enum power_management_env envs[] =
> {PM_ENV_ACPI_CPUFREQ,
> >                         PM_ENV_KVM_VM,
> > -                       PM_ENV_PSTATE_CPUFREQ};
> > +                       PM_ENV_PSTATE_CPUFREQ,
> > +                       PM_ENV_CPPC_CPUFREQ};
> >
> >         unsigned int i;
> >         for (i = 0; i < RTE_DIM(envs); ++i) { diff --git
> > a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c index
> > 0c3adc5f3..8516df4ca 100644
> > --- a/app/test/test_power_cpufreq.c
> > +++ b/app/test/test_power_cpufreq.c
> > @@ -496,7 +496,8 @@ test_power_cpufreq(void)
> >
> >         /* Test environment configuration */
> >         env = rte_power_get_env();
> > -       if ((env != PM_ENV_ACPI_CPUFREQ) && (env !=
> PM_ENV_PSTATE_CPUFREQ)) {
> > +       if ((env != PM_ENV_ACPI_CPUFREQ) && (env !=
> PM_ENV_PSTATE_CPUFREQ) &&
> > +                       (env != PM_ENV_CPPC_CPUFREQ)) {
> >                 printf("Unexpectedly got an environment other than
> ACPI/PSTATE\n");
> >                 goto fail_all;
> >         }
> > diff --git a/lib/power/meson.build b/lib/power/meson.build index
> > 74c5f3a29..4a5b07292 100644
> > --- a/lib/power/meson.build
> > +++ b/lib/power/meson.build
> > @@ -21,6 +21,7 @@ sources = files(
> >          'rte_power.c',
> >          'rte_power_empty_poll.c',
> >          'rte_power_pmd_mgmt.c',
> > +       'power_cppc_cpufreq.c',
> 
> Wrong indent + worth sorting alphabetically.
> 
> 
> >  )
> >  headers = files(
> >          'rte_power.h',
> 
> [snip]
> 
> > diff --git a/lib/power/power_cppc_cpufreq.h
> > b/lib/power/power_cppc_cpufreq.h new file mode 100644 index
> > 000000000..3bfc5f59c
> > --- /dev/null
> > +++ b/lib/power/power_cppc_cpufreq.h
> > @@ -0,0 +1,230 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2010-2021 Intel Corporation
> > + * Copyright(c) 2021 Arm Limited
> > + */
> > +
> > +#ifndef _POWER_CPPC_CPUFREQ_H
> > +#define _POWER_CPPC_CPUFREQ_H
> > +
> > +/**
> > + * @file
> > + * RTE Power Management via userspace CPPC cpufreq  */
> > +
> > +#include <rte_common.h>
> > +#include <rte_byteorder.h>
> > +#include <rte_log.h>
> > +#include <rte_string_fns.h>
> > +#include "rte_power.h"
> > +
> 
> [snip]
> 
> > +#endif
> 
> Nit: #endif /* _POWER_CPPC_CPUFREQ_H */
> 
> 
> > diff --git a/lib/power/rte_power.c b/lib/power/rte_power.c index
> > 98eaba915..3d51ff8be 100644
> > --- a/lib/power/rte_power.c
> > +++ b/lib/power/rte_power.c
> > @@ -10,6 +10,7 @@
> >  #include "power_kvm_vm.h"
> >  #include "power_pstate_cpufreq.h"
> >  #include "power_common.h"
> > +#include "power_cppc_cpufreq.h"
> 
> Nit: this include can be sorted alphabetically with other driver headers above.
> 
> 
> >
> >  enum power_management_env global_default_env = PM_ENV_NOT_SET;
> >
> 
> 
> --
> David Marchand

[dpdk-dev] [PATCH v7 0/2] power: add support for cppc cpufreq driver

[Richael Zhuang]

v7:
base on new API(open_core_sysfs_file) in http://dpdk.org/patch/95581,
update release note in release_21_08.rst.

Richael Zhuang (2):
  power: add support for cppc cpufreq
  test/power: round cpuinfo cur freq only in CPPC cpufreq

 app/test/test_power.c                  |   3 +-
 app/test/test_power_cpufreq.c          |  26 +-
 doc/guides/rel_notes/release_21_08.rst |   4 +
 lib/power/meson.build                  |   1 +
 lib/power/power_cppc_cpufreq.c         | 680 +++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h         | 230 +++++++++
 lib/power/rte_power.c                  |  26 +
 lib/power/rte_power.h                  |   2 +-
 8 files changed, 960 insertions(+), 12 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

-- 
2.20.1

[dpdk-dev] [PATCH v7 1/2] power: add support for cppc cpufreq

[Richael Zhuang]

Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
supported, which are both not available on arm64 platforms. Add
support for cppc_cpufreq driver which works on most arm64 platforms.

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
---
 app/test/test_power.c                  |   3 +-
 app/test/test_power_cpufreq.c          |   3 +-
 doc/guides/rel_notes/release_21_08.rst |   4 +
 lib/power/meson.build                  |   1 +
 lib/power/power_cppc_cpufreq.c         | 680 +++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h         | 230 +++++++++
 lib/power/rte_power.c                  |  26 +
 lib/power/rte_power.h                  |   2 +-
 8 files changed, 946 insertions(+), 3 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

diff --git a/app/test/test_power.c b/app/test/test_power.c
index da1d67c0ab..b7b5561348 100644
--- a/app/test/test_power.c
+++ b/app/test/test_power.c
@@ -133,7 +133,8 @@ test_power(void)
 	/* Perform tests for valid environments.*/
 	const enum power_management_env envs[] = {PM_ENV_ACPI_CPUFREQ,
 			PM_ENV_KVM_VM,
-			PM_ENV_PSTATE_CPUFREQ};
+			PM_ENV_PSTATE_CPUFREQ,
+			PM_ENV_CPPC_CPUFREQ};
 
 	unsigned int i;
 	for (i = 0; i < RTE_DIM(envs); ++i) {
diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index 0c3adc5f33..8516df4ca6 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -496,7 +496,8 @@ test_power_cpufreq(void)
 
 	/* Test environment configuration */
 	env = rte_power_get_env();
-	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ)) {
+	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ) &&
+			(env != PM_ENV_CPPC_CPUFREQ)) {
 		printf("Unexpectedly got an environment other than ACPI/PSTATE\n");
 		goto fail_all;
 	}
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index c92e016783..6fd9f0168a 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -103,6 +103,10 @@ New Features
   usecases. Configuration happens via standard rawdev enq/deq operations. See
   the :doc:`../rawdevs/cnxk_bphy` rawdev guide for more details on this driver.
 
+* **Added cppc_cpufreq support to Power Management library.**
+
+  Added support for cppc_cpufreq driver which works on most arm64 platforms.
+
 
 Removed Items
 -------------
diff --git a/lib/power/meson.build b/lib/power/meson.build
index c1097d32f1..36e5a65874 100644
--- a/lib/power/meson.build
+++ b/lib/power/meson.build
@@ -9,6 +9,7 @@ sources = files(
         'guest_channel.c',
         'power_acpi_cpufreq.c',
         'power_common.c',
+        'power_cppc_cpufreq.c',
         'power_kvm_vm.c',
         'power_pstate_cpufreq.c',
         'rte_power.c',
diff --git a/lib/power/power_cppc_cpufreq.c b/lib/power/power_cppc_cpufreq.c
new file mode 100644
index 0000000000..ec3293d7cf
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.c
@@ -0,0 +1,680 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2021 Intel Corporation
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#include <rte_memcpy.h>
+#include <rte_memory.h>
+
+#include "power_cppc_cpufreq.h"
+#include "power_common.h"
+
+/* macros used for rounding frequency to nearest 100000 */
+#define FREQ_ROUNDING_DELTA 50000
+#define ROUND_FREQ_TO_N_100000 100000
+
+/* the unit of highest_perf and nominal_perf differs on different arm platforms.
+ * For highest_perf, it maybe 300 or 3000000, both means 3.0GHz.
+ */
+#define UNIT_DIFF 10000
+
+#define POWER_CONVERT_TO_DECIMAL 10
+
+#define POWER_GOVERNOR_USERSPACE "userspace"
+#define POWER_SYSFILE_SETSPEED   \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_setspeed"
+#define POWER_SYSFILE_SCALING_MAX_FREQ \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
+#define POWER_SYSFILE_SCALING_MIN_FREQ  \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
+#define POWER_SYSFILE_HIGHEST_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/highest_perf"
+#define POWER_SYSFILE_NOMINAL_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/nominal_perf"
+#define POWER_SYSFILE_SYS_MAX \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
+
+#define POWER_CPPC_DRIVER "cppc-cpufreq"
+#define BUS_FREQ     100000
+
+enum power_state {
+	POWER_IDLE = 0,
+	POWER_ONGOING,
+	POWER_USED,
+	POWER_UNKNOWN
+};
+
+/**
+ * Power info per lcore.
+ */
+struct cppc_power_info {
+	unsigned int lcore_id;                   /**< Logical core id */
+	uint32_t state;                      /**< Power in use state */
+	FILE *f;                             /**< FD of scaling_setspeed */
+	char governor_ori[32];               /**< Original governor name */
+	uint32_t curr_idx;                   /**< Freq index in freqs array */
+	uint32_t highest_perf;		     /**< system wide max freq */
+	uint32_t nominal_perf;		     /**< system wide nominal freq */
+	uint16_t turbo_available;            /**< Turbo Boost available */
+	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
+	uint32_t nb_freqs;                   /**< number of available freqs */
+	uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
+} __rte_cache_aligned;
+
+static struct cppc_power_info lcore_power_info[RTE_MAX_LCORE];
+
+/**
+ * It is to set specific freq for specific logical core, according to the index
+ * of supported frequencies.
+ */
+static int
+set_freq_internal(struct cppc_power_info *pi, uint32_t idx)
+{
+	if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
+				"should be less than %u\n", idx, pi->nb_freqs);
+		return -1;
+	}
+
+	/* Check if it is the same as current */
+	if (idx == pi->curr_idx)
+		return 0;
+
+	POWER_DEBUG_TRACE("Frequency[%u] %u to be set for lcore %u\n",
+			idx, pi->freqs[idx], pi->lcore_id);
+	if (fseek(pi->f, 0, SEEK_SET) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
+			"for setting frequency for lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	if (fprintf(pi->f, "%u", pi->freqs[idx]) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new frequency for "
+				"lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	fflush(pi->f);
+	pi->curr_idx = idx;
+
+	return 1;
+}
+
+/**
+ * It is to check the current scaling governor by reading sys file, and then
+ * set it into 'userspace' if it is not by writing the sys file. The original
+ * governor will be saved for rolling back.
+ */
+static int
+power_set_governor_userspace(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, POWER_GOVERNOR_USERSPACE,
+			pi->governor_ori, sizeof(pi->governor_ori));
+}
+
+static int
+power_check_turbo(struct cppc_power_info *pi)
+{
+	FILE *f_nom, *f_max, *f_cmax;
+	int ret = -1;
+	uint32_t nominal_perf = 0, highest_perf = 0, cpuinfo_max_freq = 0;
+
+	open_core_sysfs_file(&f_max, "r", POWER_SYSFILE_HIGHEST_PERF,
+			pi->lcore_id);
+	if (f_max == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_HIGHEST_PERF);
+		goto err;
+	}
+
+	open_core_sysfs_file(&f_nom, "r", POWER_SYSFILE_NOMINAL_PERF,
+			pi->lcore_id);
+	if (f_nom == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_NOMINAL_PERF);
+		goto err;
+	}
+
+	open_core_sysfs_file(&f_cmax, "r", POWER_SYSFILE_SYS_MAX,
+			pi->lcore_id);
+	if (f_cmax == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_SYS_MAX);
+		goto err;
+	}
+
+	ret = read_core_sysfs_u32(f_max, &highest_perf);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_HIGHEST_PERF);
+		goto err;
+	}
+
+	ret = read_core_sysfs_u32(f_nom, &nominal_perf);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_NOMINAL_PERF);
+		goto err;
+	}
+
+	ret = read_core_sysfs_u32(f_cmax, &cpuinfo_max_freq);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_SYS_MAX);
+		goto err;
+	}
+
+	pi->highest_perf = highest_perf;
+	pi->nominal_perf = nominal_perf;
+
+	if ((highest_perf > nominal_perf) && ((cpuinfo_max_freq == highest_perf)
+			|| cpuinfo_max_freq == highest_perf * UNIT_DIFF)) {
+		pi->turbo_available = 1;
+		pi->turbo_enable = 1;
+		ret = 0;
+		POWER_DEBUG_TRACE("Lcore %u can do Turbo Boost! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	} else {
+		pi->turbo_available = 0;
+		pi->turbo_enable = 0;
+		POWER_DEBUG_TRACE("Lcore %u Turbo not available! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	}
+
+err:
+	if (f_max != NULL)
+		fclose(f_max);
+	if (f_nom != NULL)
+		fclose(f_nom);
+	if (f_cmax != NULL)
+		fclose(f_cmax);
+
+	return ret;
+}
+
+/**
+ * It is to get the available frequencies of the specific lcore by reading the
+ * sys file.
+ */
+static int
+power_get_available_freqs(struct cppc_power_info *pi)
+{
+	FILE *f_min, *f_max;
+	int ret = -1;
+	uint32_t scaling_min_freq = 0, scaling_max_freq = 0, nominal_perf = 0;
+	uint32_t i, num_freqs = 0;
+
+	open_core_sysfs_file(&f_max, "r", POWER_SYSFILE_SCALING_MAX_FREQ,
+			pi->lcore_id);
+	if (f_max == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_SCALING_MAX_FREQ);
+		goto out;
+	}
+
+	open_core_sysfs_file(&f_min, "r", POWER_SYSFILE_SCALING_MIN_FREQ,
+			pi->lcore_id);
+	if (f_min == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_SCALING_MIN_FREQ);
+		goto out;
+	}
+
+	ret = read_core_sysfs_u32(f_max, &scaling_max_freq);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_SCALING_MAX_FREQ);
+		goto out;
+	}
+
+	ret = read_core_sysfs_u32(f_min, &scaling_min_freq);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_SCALING_MIN_FREQ);
+		goto out;
+	}
+
+	power_check_turbo(pi);
+
+	if (scaling_max_freq < scaling_min_freq)
+		goto out;
+
+	/* If turbo is available then there is one extra freq bucket
+	 * to store the sys max freq which value is scaling_max_freq
+	 */
+	nominal_perf = (pi->nominal_perf < UNIT_DIFF) ?
+			pi->nominal_perf * UNIT_DIFF : pi->nominal_perf;
+	num_freqs = (nominal_perf - scaling_min_freq) / BUS_FREQ + 1 +
+		pi->turbo_available;
+
+	/* Generate the freq bucket array. */
+	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
+		if ((i == 0) && pi->turbo_available)
+			pi->freqs[pi->nb_freqs++] = scaling_max_freq;
+		else
+			pi->freqs[pi->nb_freqs++] =
+			nominal_perf - (i - pi->turbo_available) * BUS_FREQ;
+	}
+
+	ret = 0;
+
+	POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
+			num_freqs, pi->lcore_id);
+
+out:
+	if (f_min != NULL)
+		fclose(f_min);
+	if (f_max != NULL)
+		fclose(f_max);
+
+	return ret;
+}
+
+/**
+ * It is to fopen the sys file for the future setting the lcore frequency.
+ */
+static int
+power_init_for_setting_freq(struct cppc_power_info *pi)
+{
+	FILE *f;
+	char buf[BUFSIZ];
+	uint32_t i, freq;
+	int ret;
+
+	open_core_sysfs_file(&f, "rw+", POWER_SYSFILE_SETSPEED, pi->lcore_id);
+	if (f == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_SETSPEED);
+		goto err;
+	}
+
+	ret = read_core_sysfs_s(f, buf, sizeof(buf));
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_SETSPEED);
+		goto err;
+	}
+
+	freq = strtoul(buf, NULL, POWER_CONVERT_TO_DECIMAL);
+
+	/* convert the frequency to nearest 100000 value
+	 * Ex: if freq=1396789 then freq_conv=1400000
+	 * Ex: if freq=800030 then freq_conv=800000
+	 */
+	unsigned int freq_conv = 0;
+	freq_conv = (freq + FREQ_ROUNDING_DELTA)
+				/ ROUND_FREQ_TO_N_100000;
+	freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
+
+	for (i = 0; i < pi->nb_freqs; i++) {
+		if (freq_conv == pi->freqs[i]) {
+			pi->curr_idx = i;
+			pi->f = f;
+			return 0;
+		}
+	}
+
+err:
+	if (f != NULL)
+		fclose(f);
+
+	return -1;
+}
+
+int
+power_cppc_cpufreq_check_supported(void)
+{
+	return cpufreq_check_scaling_driver(POWER_CPPC_DRIVER);
+}
+
+int
+power_cppc_cpufreq_init(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_IDLE;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"in use\n", lcore_id);
+		return -1;
+	}
+
+	pi->lcore_id = lcore_id;
+	/* Check and set the governor */
+	if (power_set_governor_userspace(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
+				"userspace\n", lcore_id);
+		goto fail;
+	}
+
+	/* Get the available frequencies */
+	if (power_get_available_freqs(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Init for setting lcore frequency */
+	if (power_init_for_setting_freq(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Set freq to max by default */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
+				"to max\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
+			"power management\n", lcore_id);
+
+	__atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	return -1;
+}
+
+/**
+ * It is to check the governor and then set the original governor back if
+ * needed by writing the sys file.
+ */
+static int
+power_set_governor_original(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, pi->governor_ori, NULL, 0);
+}
+
+int
+power_cppc_cpufreq_exit(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_USED;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"not used\n", lcore_id);
+		return -1;
+	}
+
+	/* Close FD of setting freq */
+	fclose(pi->f);
+	pi->f = NULL;
+
+	/* Set the governor back to the original */
+	if (power_set_governor_original(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
+				"to the original\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
+			"'userspace' mode and been set back to the "
+			"original\n", lcore_id);
+	__atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+
+	return -1;
+}
+
+uint32_t
+power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return 0;
+	}
+
+	if (freqs == NULL) {
+		RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
+		return 0;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (num < pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
+		return 0;
+	}
+	rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
+
+	return pi->nb_freqs;
+}
+
+uint32_t
+power_cppc_cpufreq_get_freq(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return RTE_POWER_INVALID_FREQ_INDEX;
+	}
+
+	return lcore_power_info[lcore_id].curr_idx;
+}
+
+int
+power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	return set_freq_internal(&(lcore_power_info[lcore_id]), index);
+}
+
+int
+power_cppc_cpufreq_freq_down(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx + 1 == pi->nb_freqs)
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx + 1);
+}
+
+int
+power_cppc_cpufreq_freq_up(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx == 0 || (pi->curr_idx == 1 &&
+		pi->turbo_available && !pi->turbo_enable))
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx - 1);
+}
+
+int
+power_cppc_cpufreq_freq_max(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	/* Frequencies in the array are from high to low. */
+	if (lcore_power_info[lcore_id].turbo_available) {
+		if (lcore_power_info[lcore_id].turbo_enable)
+			/* Set to Turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 0);
+		else
+			/* Set to max non-turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 1);
+	} else
+		return set_freq_internal(&lcore_power_info[lcore_id], 0);
+}
+
+int
+power_cppc_cpufreq_freq_min(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->nb_freqs - 1);
+}
+
+int
+power_cppc_turbo_status(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	return pi->turbo_enable;
+}
+
+int
+power_cppc_enable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	if (pi->turbo_available)
+		pi->turbo_enable = 1;
+	else {
+		pi->turbo_enable = 0;
+		RTE_LOG(ERR, POWER,
+			"Failed to enable turbo on lcore %u\n",
+			lcore_id);
+		return -1;
+	}
+
+	/* TODO: must set to max once enbling Turbo? Considering add condition:
+	 * if ((pi->turbo_available) && (pi->curr_idx <= 1))
+	 */
+	/* Max may have changed, so call to max function */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER,
+			"Failed to set frequency of lcore %u to max\n",
+			lcore_id);
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+power_cppc_disable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	pi->turbo_enable = 0;
+
+	if ((pi->turbo_available) && (pi->curr_idx <= 1)) {
+		/* Try to set freq to max by default coming out of turbo */
+		if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+			RTE_LOG(ERR, POWER,
+				"Failed to set frequency of lcore %u to max\n",
+				lcore_id);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+int
+power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+	if (caps == NULL) {
+		RTE_LOG(ERR, POWER, "Invalid argument\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	caps->capabilities = 0;
+	caps->turbo = !!(pi->turbo_available);
+
+	return 0;
+}
diff --git a/lib/power/power_cppc_cpufreq.h b/lib/power/power_cppc_cpufreq.h
new file mode 100644
index 0000000000..57c0ee880c
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.h
@@ -0,0 +1,230 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2021 Intel Corporation
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#ifndef _POWER_CPPC_CPUFREQ_H
+#define _POWER_CPPC_CPUFREQ_H
+
+/**
+ * @file
+ * RTE Power Management via userspace CPPC cpufreq
+ */
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_string_fns.h>
+#include "rte_power.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Check if CPPC power management is supported.
+ *
+ * @return
+ *   - 1 if supported
+ *   - 0 if unsupported
+ *   - -1 if error, with rte_errno indicating reason for error.
+ */
+int power_cppc_cpufreq_check_supported(void);
+
+/**
+ * Initialize power management for a specific lcore. It will check and set the
+ * governor to userspace for the lcore, get the available frequencies, and
+ * prepare to set new lcore frequency.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_init(unsigned int lcore_id);
+
+/**
+ * Exit power management on a specific lcore. It will set the governor to which
+ * is before initialized.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_exit(unsigned int lcore_id);
+
+/**
+ * Get the available frequencies of a specific lcore. The return value will be
+ * the minimal one of the total number of available frequencies and the number
+ * of buffer. The index of available frequencies used in other interfaces
+ * should be in the range of 0 to this return value.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param freqs
+ *  The buffer array to save the frequencies.
+ * @param num
+ *  The number of frequencies to get.
+ *
+ * @return
+ *  The number of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs,
+		uint32_t num);
+
+/**
+ * Return the current index of available frequencies of a specific lcore. It
+ * will return 'RTE_POWER_INVALID_FREQ_INDEX = (~0)' if error.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  The current index of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_get_freq(unsigned int lcore_id);
+
+/**
+ * Set the new frequency for a specific lcore by indicating the index of
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param index
+ *  The index of available frequencies.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index);
+
+/**
+ * Scale up the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_up(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_down(unsigned int lcore_id);
+
+/**
+ * Scale up the frequency of a specific lcore to the highest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_max(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore to the lowest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_min(unsigned int lcore_id);
+
+/**
+ * Get the turbo status of a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 Turbo Boost is enabled on this lcore.
+ *  - 0 Turbo Boost is disabled on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_turbo_status(unsigned int lcore_id);
+
+/**
+ * Enable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost is enabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_enable_turbo(unsigned int lcore_id);
+
+/**
+ * Disable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost disabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_disable_turbo(unsigned int lcore_id);
+
+/**
+ * Returns power capabilities for a specific lcore.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param caps
+ *  pointer to rte_power_core_capabilities object.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _POWER_CPPC_CPUFREQ_H */
diff --git a/lib/power/rte_power.c b/lib/power/rte_power.c
index 98eaba9154..3cba56bac9 100644
--- a/lib/power/rte_power.c
+++ b/lib/power/rte_power.c
@@ -7,6 +7,7 @@
 
 #include "rte_power.h"
 #include "power_acpi_cpufreq.h"
+#include "power_cppc_cpufreq.h"
 #include "power_kvm_vm.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
@@ -54,6 +55,8 @@ rte_power_check_env_supported(enum power_management_env env)
 		return power_pstate_cpufreq_check_supported();
 	case PM_ENV_KVM_VM:
 		return power_kvm_vm_check_supported();
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_check_supported();
 	default:
 		rte_errno = EINVAL;
 		return -1;
@@ -110,6 +113,18 @@ rte_power_set_env(enum power_management_env env)
 		rte_power_freq_disable_turbo = power_pstate_disable_turbo;
 		rte_power_get_capabilities = power_pstate_get_capabilities;
 
+	} else if (env == PM_ENV_CPPC_CPUFREQ) {
+		rte_power_freqs = power_cppc_cpufreq_freqs;
+		rte_power_get_freq = power_cppc_cpufreq_get_freq;
+		rte_power_set_freq = power_cppc_cpufreq_set_freq;
+		rte_power_freq_up = power_cppc_cpufreq_freq_up;
+		rte_power_freq_down = power_cppc_cpufreq_freq_down;
+		rte_power_freq_min = power_cppc_cpufreq_freq_min;
+		rte_power_freq_max = power_cppc_cpufreq_freq_max;
+		rte_power_turbo_status = power_cppc_turbo_status;
+		rte_power_freq_enable_turbo = power_cppc_enable_turbo;
+		rte_power_freq_disable_turbo = power_cppc_disable_turbo;
+		rte_power_get_capabilities = power_cppc_get_capabilities;
 	} else {
 		RTE_LOG(ERR, POWER, "Invalid Power Management Environment(%d) set\n",
 				env);
@@ -153,6 +168,8 @@ rte_power_init(unsigned int lcore_id)
 		return power_kvm_vm_init(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_init(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_init(lcore_id);
 	default:
 		RTE_LOG(INFO, POWER, "Env isn't set yet!\n");
 	}
@@ -172,6 +189,13 @@ rte_power_init(unsigned int lcore_id)
 		goto out;
 	}
 
+	RTE_LOG(INFO, POWER, "Attempting to initialise CPPC power management...\n");
+	ret = power_cppc_cpufreq_init(lcore_id);
+	if (ret == 0) {
+		rte_power_set_env(PM_ENV_CPPC_CPUFREQ);
+		goto out;
+	}
+
 	RTE_LOG(INFO, POWER, "Attempting to initialise VM power management...\n");
 	ret = power_kvm_vm_init(lcore_id);
 	if (ret == 0) {
@@ -194,6 +218,8 @@ rte_power_exit(unsigned int lcore_id)
 		return power_kvm_vm_exit(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_exit(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_exit(lcore_id);
 	default:
 		RTE_LOG(ERR, POWER, "Environment has not been set, unable to exit gracefully\n");
 
diff --git a/lib/power/rte_power.h b/lib/power/rte_power.h
index c8086bf6ba..c5759afa39 100644
--- a/lib/power/rte_power.h
+++ b/lib/power/rte_power.h
@@ -22,7 +22,7 @@ extern "C" {
 
 /* Power Management Environment State */
 enum power_management_env {PM_ENV_NOT_SET, PM_ENV_ACPI_CPUFREQ, PM_ENV_KVM_VM,
-		PM_ENV_PSTATE_CPUFREQ};
+		PM_ENV_PSTATE_CPUFREQ, PM_ENV_CPPC_CPUFREQ};
 
 /**
  * @warning
-- 
2.20.1

Re: [dpdk-dev] [PATCH v7 1/2] power: add support for cppc cpufreq

[David Marchand]

On Fri, Jul 9, 2021 at 5:34 AM Richael Zhuang <richael.zhuang@arm.com> wrote:
>
> Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> supported, which are both not available on arm64 platforms. Add
> support for cppc_cpufreq driver which works on most arm64 platforms.

GHA reported build issues with clang.
https://github.com/ovsrobot/dpdk/runs/3025478784?check_suite_focus=true#step:15:787

Could you look at it?
Thanks.


-- 
David Marchand

Re: [dpdk-dev] [PATCH v7 1/2] power: add support for cppc cpufreq

[Richael Zhuang]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Friday, July 9, 2021 5:10 PM
> To: Richael Zhuang <Richael.Zhuang@arm.com>
> Cc: dev <dev@dpdk.org>; Yu Jiang <yux.jiang@intel.com>; David Hunt
> <david.hunt@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v7 1/2] power: add support for cppc cpufreq
> 
> On Fri, Jul 9, 2021 at 5:34 AM Richael Zhuang <richael.zhuang@arm.com>
> wrote:
> >
> > Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> > supported, which are both not available on arm64 platforms. Add
> > support for cppc_cpufreq driver which works on most arm64 platforms.
> 
> GHA reported build issues with clang.
> https://github.com/ovsrobot/dpdk/runs/3025478784?check_suite_focus=tru
> e#step:15:787
> 
> Could you look at it?
> Thanks.
> 
> --
> David Marchand

OK, I will fix it soon.

Re: [dpdk-dev] [PATCH v7 1/2] power: add support for cppc cpufreq

[David Hunt]

On 9/7/2021 4:34 AM, Richael Zhuang wrote:
> Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
> supported, which are both not available on arm64 platforms. Add
> support for cppc_cpufreq driver which works on most arm64 platforms.
>
> Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> ---


--snip--

Hi Richael,

It's usuall to inherit the tags from previous versions (such as 
Acked-by:, Tested-by:, etc.) if the changes are relatively minor. That 
helps the patchwork flags keep up to date.

And by "inherit" I mean you can add them in to your patch just below the 
Signed-off-by: tag.

If it's a major re-work of a patch, then there might need to be 
re-review or re-ack.

If you’re doing a re-spin of this patch set for the clang issue, that 
might be a good opportunity to include the tags from previous versions.

Rgds,
Dave.

[dpdk-dev] [PATCH v8 0/2] power: add support for cppc cpufreq driver

[Richael Zhuang]

v8:
fix build issue with clang

Richael Zhuang (2):
  power: add support for cppc cpufreq
  test/power: round cpuinfo cur freq only in CPPC cpufreq

 app/test/test_power.c                  |   3 +-
 app/test/test_power_cpufreq.c          |  26 +-
 doc/guides/rel_notes/release_21_08.rst |   4 +
 lib/power/meson.build                  |   1 +
 lib/power/power_cppc_cpufreq.c         | 680 +++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h         | 230 +++++++++
 lib/power/rte_power.c                  |  26 +
 lib/power/rte_power.h                  |   2 +-
 8 files changed, 960 insertions(+), 12 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

-- 
2.20.1

[dpdk-dev] [PATCH v8 1/2] power: add support for cppc cpufreq

[Richael Zhuang]

Currently in DPDK only acpi_cpufreq and pstate_cpufreq drivers are
supported, which are both not available on arm64 platforms. Add
support for cppc_cpufreq driver which works on most arm64 platforms.

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
Acked-by: David Hunt <david.hunt@intel.com>

---
 app/test/test_power.c                  |   3 +-
 app/test/test_power_cpufreq.c          |   3 +-
 doc/guides/rel_notes/release_21_08.rst |   4 +
 lib/power/meson.build                  |   1 +
 lib/power/power_cppc_cpufreq.c         | 680 +++++++++++++++++++++++++
 lib/power/power_cppc_cpufreq.h         | 230 +++++++++
 lib/power/rte_power.c                  |  26 +
 lib/power/rte_power.h                  |   2 +-
 8 files changed, 946 insertions(+), 3 deletions(-)
 create mode 100644 lib/power/power_cppc_cpufreq.c
 create mode 100644 lib/power/power_cppc_cpufreq.h

diff --git a/app/test/test_power.c b/app/test/test_power.c
index da1d67c0ab..b7b5561348 100644
--- a/app/test/test_power.c
+++ b/app/test/test_power.c
@@ -133,7 +133,8 @@ test_power(void)
 	/* Perform tests for valid environments.*/
 	const enum power_management_env envs[] = {PM_ENV_ACPI_CPUFREQ,
 			PM_ENV_KVM_VM,
-			PM_ENV_PSTATE_CPUFREQ};
+			PM_ENV_PSTATE_CPUFREQ,
+			PM_ENV_CPPC_CPUFREQ};
 
 	unsigned int i;
 	for (i = 0; i < RTE_DIM(envs); ++i) {
diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index 0c3adc5f33..8516df4ca6 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -496,7 +496,8 @@ test_power_cpufreq(void)
 
 	/* Test environment configuration */
 	env = rte_power_get_env();
-	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ)) {
+	if ((env != PM_ENV_ACPI_CPUFREQ) && (env != PM_ENV_PSTATE_CPUFREQ) &&
+			(env != PM_ENV_CPPC_CPUFREQ)) {
 		printf("Unexpectedly got an environment other than ACPI/PSTATE\n");
 		goto fail_all;
 	}
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index c92e016783..6fd9f0168a 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -103,6 +103,10 @@ New Features
   usecases. Configuration happens via standard rawdev enq/deq operations. See
   the :doc:`../rawdevs/cnxk_bphy` rawdev guide for more details on this driver.
 
+* **Added cppc_cpufreq support to Power Management library.**
+
+  Added support for cppc_cpufreq driver which works on most arm64 platforms.
+
 
 Removed Items
 -------------
diff --git a/lib/power/meson.build b/lib/power/meson.build
index c1097d32f1..36e5a65874 100644
--- a/lib/power/meson.build
+++ b/lib/power/meson.build
@@ -9,6 +9,7 @@ sources = files(
         'guest_channel.c',
         'power_acpi_cpufreq.c',
         'power_common.c',
+        'power_cppc_cpufreq.c',
         'power_kvm_vm.c',
         'power_pstate_cpufreq.c',
         'rte_power.c',
diff --git a/lib/power/power_cppc_cpufreq.c b/lib/power/power_cppc_cpufreq.c
new file mode 100644
index 0000000000..e92973ab54
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.c
@@ -0,0 +1,680 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2021 Intel Corporation
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#include <rte_memcpy.h>
+#include <rte_memory.h>
+
+#include "power_cppc_cpufreq.h"
+#include "power_common.h"
+
+/* macros used for rounding frequency to nearest 100000 */
+#define FREQ_ROUNDING_DELTA 50000
+#define ROUND_FREQ_TO_N_100000 100000
+
+/* the unit of highest_perf and nominal_perf differs on different arm platforms.
+ * For highest_perf, it maybe 300 or 3000000, both means 3.0GHz.
+ */
+#define UNIT_DIFF 10000
+
+#define POWER_CONVERT_TO_DECIMAL 10
+
+#define POWER_GOVERNOR_USERSPACE "userspace"
+#define POWER_SYSFILE_SETSPEED   \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_setspeed"
+#define POWER_SYSFILE_SCALING_MAX_FREQ \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
+#define POWER_SYSFILE_SCALING_MIN_FREQ  \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
+#define POWER_SYSFILE_HIGHEST_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/highest_perf"
+#define POWER_SYSFILE_NOMINAL_PERF \
+		"/sys/devices/system/cpu/cpu%u/acpi_cppc/nominal_perf"
+#define POWER_SYSFILE_SYS_MAX \
+		"/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
+
+#define POWER_CPPC_DRIVER "cppc-cpufreq"
+#define BUS_FREQ     100000
+
+enum power_state {
+	POWER_IDLE = 0,
+	POWER_ONGOING,
+	POWER_USED,
+	POWER_UNKNOWN
+};
+
+/**
+ * Power info per lcore.
+ */
+struct cppc_power_info {
+	unsigned int lcore_id;                   /**< Logical core id */
+	uint32_t state;                      /**< Power in use state */
+	FILE *f;                             /**< FD of scaling_setspeed */
+	char governor_ori[32];               /**< Original governor name */
+	uint32_t curr_idx;                   /**< Freq index in freqs array */
+	uint32_t highest_perf;		     /**< system wide max freq */
+	uint32_t nominal_perf;		     /**< system wide nominal freq */
+	uint16_t turbo_available;            /**< Turbo Boost available */
+	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
+	uint32_t nb_freqs;                   /**< number of available freqs */
+	uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
+} __rte_cache_aligned;
+
+static struct cppc_power_info lcore_power_info[RTE_MAX_LCORE];
+
+/**
+ * It is to set specific freq for specific logical core, according to the index
+ * of supported frequencies.
+ */
+static int
+set_freq_internal(struct cppc_power_info *pi, uint32_t idx)
+{
+	if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
+				"should be less than %u\n", idx, pi->nb_freqs);
+		return -1;
+	}
+
+	/* Check if it is the same as current */
+	if (idx == pi->curr_idx)
+		return 0;
+
+	POWER_DEBUG_TRACE("Frequency[%u] %u to be set for lcore %u\n",
+			idx, pi->freqs[idx], pi->lcore_id);
+	if (fseek(pi->f, 0, SEEK_SET) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
+			"for setting frequency for lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	if (fprintf(pi->f, "%u", pi->freqs[idx]) < 0) {
+		RTE_LOG(ERR, POWER, "Fail to write new frequency for "
+				"lcore %u\n", pi->lcore_id);
+		return -1;
+	}
+	fflush(pi->f);
+	pi->curr_idx = idx;
+
+	return 1;
+}
+
+/**
+ * It is to check the current scaling governor by reading sys file, and then
+ * set it into 'userspace' if it is not by writing the sys file. The original
+ * governor will be saved for rolling back.
+ */
+static int
+power_set_governor_userspace(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, POWER_GOVERNOR_USERSPACE,
+			pi->governor_ori, sizeof(pi->governor_ori));
+}
+
+static int
+power_check_turbo(struct cppc_power_info *pi)
+{
+	FILE *f_nom = NULL, *f_max = NULL, *f_cmax = NULL;
+	int ret = -1;
+	uint32_t nominal_perf = 0, highest_perf = 0, cpuinfo_max_freq = 0;
+
+	open_core_sysfs_file(&f_max, "r", POWER_SYSFILE_HIGHEST_PERF,
+			pi->lcore_id);
+	if (f_max == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_HIGHEST_PERF);
+		goto err;
+	}
+
+	open_core_sysfs_file(&f_nom, "r", POWER_SYSFILE_NOMINAL_PERF,
+			pi->lcore_id);
+	if (f_nom == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_NOMINAL_PERF);
+		goto err;
+	}
+
+	open_core_sysfs_file(&f_cmax, "r", POWER_SYSFILE_SYS_MAX,
+			pi->lcore_id);
+	if (f_cmax == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_SYS_MAX);
+		goto err;
+	}
+
+	ret = read_core_sysfs_u32(f_max, &highest_perf);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_HIGHEST_PERF);
+		goto err;
+	}
+
+	ret = read_core_sysfs_u32(f_nom, &nominal_perf);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_NOMINAL_PERF);
+		goto err;
+	}
+
+	ret = read_core_sysfs_u32(f_cmax, &cpuinfo_max_freq);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_SYS_MAX);
+		goto err;
+	}
+
+	pi->highest_perf = highest_perf;
+	pi->nominal_perf = nominal_perf;
+
+	if ((highest_perf > nominal_perf) && ((cpuinfo_max_freq == highest_perf)
+			|| cpuinfo_max_freq == highest_perf * UNIT_DIFF)) {
+		pi->turbo_available = 1;
+		pi->turbo_enable = 1;
+		ret = 0;
+		POWER_DEBUG_TRACE("Lcore %u can do Turbo Boost! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	} else {
+		pi->turbo_available = 0;
+		pi->turbo_enable = 0;
+		POWER_DEBUG_TRACE("Lcore %u Turbo not available! highest perf %u, "
+				"nominal perf %u\n",
+				pi->lcore_id, highest_perf, nominal_perf);
+	}
+
+err:
+	if (f_max != NULL)
+		fclose(f_max);
+	if (f_nom != NULL)
+		fclose(f_nom);
+	if (f_cmax != NULL)
+		fclose(f_cmax);
+
+	return ret;
+}
+
+/**
+ * It is to get the available frequencies of the specific lcore by reading the
+ * sys file.
+ */
+static int
+power_get_available_freqs(struct cppc_power_info *pi)
+{
+	FILE *f_min = NULL, *f_max = NULL;
+	int ret = -1;
+	uint32_t scaling_min_freq = 0, scaling_max_freq = 0, nominal_perf = 0;
+	uint32_t i, num_freqs = 0;
+
+	open_core_sysfs_file(&f_max, "r", POWER_SYSFILE_SCALING_MAX_FREQ,
+			pi->lcore_id);
+	if (f_max == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_SCALING_MAX_FREQ);
+		goto out;
+	}
+
+	open_core_sysfs_file(&f_min, "r", POWER_SYSFILE_SCALING_MIN_FREQ,
+			pi->lcore_id);
+	if (f_min == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_SCALING_MIN_FREQ);
+		goto out;
+	}
+
+	ret = read_core_sysfs_u32(f_max, &scaling_max_freq);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_SCALING_MAX_FREQ);
+		goto out;
+	}
+
+	ret = read_core_sysfs_u32(f_min, &scaling_min_freq);
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_SCALING_MIN_FREQ);
+		goto out;
+	}
+
+	power_check_turbo(pi);
+
+	if (scaling_max_freq < scaling_min_freq)
+		goto out;
+
+	/* If turbo is available then there is one extra freq bucket
+	 * to store the sys max freq which value is scaling_max_freq
+	 */
+	nominal_perf = (pi->nominal_perf < UNIT_DIFF) ?
+			pi->nominal_perf * UNIT_DIFF : pi->nominal_perf;
+	num_freqs = (nominal_perf - scaling_min_freq) / BUS_FREQ + 1 +
+		pi->turbo_available;
+
+	/* Generate the freq bucket array. */
+	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
+		if ((i == 0) && pi->turbo_available)
+			pi->freqs[pi->nb_freqs++] = scaling_max_freq;
+		else
+			pi->freqs[pi->nb_freqs++] =
+			nominal_perf - (i - pi->turbo_available) * BUS_FREQ;
+	}
+
+	ret = 0;
+
+	POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
+			num_freqs, pi->lcore_id);
+
+out:
+	if (f_min != NULL)
+		fclose(f_min);
+	if (f_max != NULL)
+		fclose(f_max);
+
+	return ret;
+}
+
+/**
+ * It is to fopen the sys file for the future setting the lcore frequency.
+ */
+static int
+power_init_for_setting_freq(struct cppc_power_info *pi)
+{
+	FILE *f = NULL;
+	char buf[BUFSIZ];
+	uint32_t i, freq;
+	int ret;
+
+	open_core_sysfs_file(&f, "rw+", POWER_SYSFILE_SETSPEED, pi->lcore_id);
+	if (f == NULL) {
+		RTE_LOG(ERR, POWER, "failed to open %s\n",
+				POWER_SYSFILE_SETSPEED);
+		goto err;
+	}
+
+	ret = read_core_sysfs_s(f, buf, sizeof(buf));
+	if (ret < 0) {
+		RTE_LOG(ERR, POWER, "Failed to read %s\n",
+				POWER_SYSFILE_SETSPEED);
+		goto err;
+	}
+
+	freq = strtoul(buf, NULL, POWER_CONVERT_TO_DECIMAL);
+
+	/* convert the frequency to nearest 100000 value
+	 * Ex: if freq=1396789 then freq_conv=1400000
+	 * Ex: if freq=800030 then freq_conv=800000
+	 */
+	unsigned int freq_conv = 0;
+	freq_conv = (freq + FREQ_ROUNDING_DELTA)
+				/ ROUND_FREQ_TO_N_100000;
+	freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
+
+	for (i = 0; i < pi->nb_freqs; i++) {
+		if (freq_conv == pi->freqs[i]) {
+			pi->curr_idx = i;
+			pi->f = f;
+			return 0;
+		}
+	}
+
+err:
+	if (f != NULL)
+		fclose(f);
+
+	return -1;
+}
+
+int
+power_cppc_cpufreq_check_supported(void)
+{
+	return cpufreq_check_scaling_driver(POWER_CPPC_DRIVER);
+}
+
+int
+power_cppc_cpufreq_init(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_IDLE;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"in use\n", lcore_id);
+		return -1;
+	}
+
+	pi->lcore_id = lcore_id;
+	/* Check and set the governor */
+	if (power_set_governor_userspace(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
+				"userspace\n", lcore_id);
+		goto fail;
+	}
+
+	/* Get the available frequencies */
+	if (power_get_available_freqs(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Init for setting lcore frequency */
+	if (power_init_for_setting_freq(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
+				"lcore %u\n", lcore_id);
+		goto fail;
+	}
+
+	/* Set freq to max by default */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
+				"to max\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
+			"power management\n", lcore_id);
+
+	__atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	return -1;
+}
+
+/**
+ * It is to check the governor and then set the original governor back if
+ * needed by writing the sys file.
+ */
+static int
+power_set_governor_original(struct cppc_power_info *pi)
+{
+	return power_set_governor(pi->lcore_id, pi->governor_ori, NULL, 0);
+}
+
+int
+power_cppc_cpufreq_exit(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+	uint32_t exp_state;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
+				lcore_id, RTE_MAX_LCORE - 1U);
+		return -1;
+	}
+	pi = &lcore_power_info[lcore_id];
+	exp_state = POWER_USED;
+	/* The power in use state works as a guard variable between
+	 * the CPU frequency control initialization and exit process.
+	 * The ACQUIRE memory ordering here pairs with the RELEASE
+	 * ordering below as lock to make sure the frequency operations
+	 * in the critical section are done under the correct state.
+	 */
+	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
+					POWER_ONGOING, 0,
+					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
+				"not used\n", lcore_id);
+		return -1;
+	}
+
+	/* Close FD of setting freq */
+	fclose(pi->f);
+	pi->f = NULL;
+
+	/* Set the governor back to the original */
+	if (power_set_governor_original(pi) < 0) {
+		RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
+				"to the original\n", lcore_id);
+		goto fail;
+	}
+
+	RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
+			"'userspace' mode and been set back to the "
+			"original\n", lcore_id);
+	__atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
+
+	return 0;
+
+fail:
+	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+
+	return -1;
+}
+
+uint32_t
+power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return 0;
+	}
+
+	if (freqs == NULL) {
+		RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
+		return 0;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (num < pi->nb_freqs) {
+		RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
+		return 0;
+	}
+	rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
+
+	return pi->nb_freqs;
+}
+
+uint32_t
+power_cppc_cpufreq_get_freq(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return RTE_POWER_INVALID_FREQ_INDEX;
+	}
+
+	return lcore_power_info[lcore_id].curr_idx;
+}
+
+int
+power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	return set_freq_internal(&(lcore_power_info[lcore_id]), index);
+}
+
+int
+power_cppc_cpufreq_freq_down(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx + 1 == pi->nb_freqs)
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx + 1);
+}
+
+int
+power_cppc_cpufreq_freq_up(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	if (pi->curr_idx == 0 || (pi->curr_idx == 1 &&
+		pi->turbo_available && !pi->turbo_enable))
+		return 0;
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->curr_idx - 1);
+}
+
+int
+power_cppc_cpufreq_freq_max(unsigned int lcore_id)
+{
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	/* Frequencies in the array are from high to low. */
+	if (lcore_power_info[lcore_id].turbo_available) {
+		if (lcore_power_info[lcore_id].turbo_enable)
+			/* Set to Turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 0);
+		else
+			/* Set to max non-turbo */
+			return set_freq_internal(
+				&lcore_power_info[lcore_id], 1);
+	} else
+		return set_freq_internal(&lcore_power_info[lcore_id], 0);
+}
+
+int
+power_cppc_cpufreq_freq_min(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	/* Frequencies in the array are from high to low. */
+	return set_freq_internal(pi, pi->nb_freqs - 1);
+}
+
+int
+power_cppc_turbo_status(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	return pi->turbo_enable;
+}
+
+int
+power_cppc_enable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	if (pi->turbo_available)
+		pi->turbo_enable = 1;
+	else {
+		pi->turbo_enable = 0;
+		RTE_LOG(ERR, POWER,
+			"Failed to enable turbo on lcore %u\n",
+			lcore_id);
+		return -1;
+	}
+
+	/* TODO: must set to max once enbling Turbo? Considering add condition:
+	 * if ((pi->turbo_available) && (pi->curr_idx <= 1))
+	 */
+	/* Max may have changed, so call to max function */
+	if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+		RTE_LOG(ERR, POWER,
+			"Failed to set frequency of lcore %u to max\n",
+			lcore_id);
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+power_cppc_disable_turbo(unsigned int lcore_id)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+
+	pi->turbo_enable = 0;
+
+	if ((pi->turbo_available) && (pi->curr_idx <= 1)) {
+		/* Try to set freq to max by default coming out of turbo */
+		if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
+			RTE_LOG(ERR, POWER,
+				"Failed to set frequency of lcore %u to max\n",
+				lcore_id);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+int
+power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps)
+{
+	struct cppc_power_info *pi;
+
+	if (lcore_id >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
+		return -1;
+	}
+	if (caps == NULL) {
+		RTE_LOG(ERR, POWER, "Invalid argument\n");
+		return -1;
+	}
+
+	pi = &lcore_power_info[lcore_id];
+	caps->capabilities = 0;
+	caps->turbo = !!(pi->turbo_available);
+
+	return 0;
+}
diff --git a/lib/power/power_cppc_cpufreq.h b/lib/power/power_cppc_cpufreq.h
new file mode 100644
index 0000000000..57c0ee880c
--- /dev/null
+++ b/lib/power/power_cppc_cpufreq.h
@@ -0,0 +1,230 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2021 Intel Corporation
+ * Copyright(c) 2021 Arm Limited
+ */
+
+#ifndef _POWER_CPPC_CPUFREQ_H
+#define _POWER_CPPC_CPUFREQ_H
+
+/**
+ * @file
+ * RTE Power Management via userspace CPPC cpufreq
+ */
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_string_fns.h>
+#include "rte_power.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Check if CPPC power management is supported.
+ *
+ * @return
+ *   - 1 if supported
+ *   - 0 if unsupported
+ *   - -1 if error, with rte_errno indicating reason for error.
+ */
+int power_cppc_cpufreq_check_supported(void);
+
+/**
+ * Initialize power management for a specific lcore. It will check and set the
+ * governor to userspace for the lcore, get the available frequencies, and
+ * prepare to set new lcore frequency.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_init(unsigned int lcore_id);
+
+/**
+ * Exit power management on a specific lcore. It will set the governor to which
+ * is before initialized.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_exit(unsigned int lcore_id);
+
+/**
+ * Get the available frequencies of a specific lcore. The return value will be
+ * the minimal one of the total number of available frequencies and the number
+ * of buffer. The index of available frequencies used in other interfaces
+ * should be in the range of 0 to this return value.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param freqs
+ *  The buffer array to save the frequencies.
+ * @param num
+ *  The number of frequencies to get.
+ *
+ * @return
+ *  The number of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs,
+		uint32_t num);
+
+/**
+ * Return the current index of available frequencies of a specific lcore. It
+ * will return 'RTE_POWER_INVALID_FREQ_INDEX = (~0)' if error.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  The current index of available frequencies.
+ */
+uint32_t power_cppc_cpufreq_get_freq(unsigned int lcore_id);
+
+/**
+ * Set the new frequency for a specific lcore by indicating the index of
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param index
+ *  The index of available frequencies.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index);
+
+/**
+ * Scale up the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_up(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore according to the available
+ * frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_down(unsigned int lcore_id);
+
+/**
+ * Scale up the frequency of a specific lcore to the highest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_max(unsigned int lcore_id);
+
+/**
+ * Scale down the frequency of a specific lcore to the lowest according to the
+ * available frequencies.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 on success with frequency changed.
+ *  - 0 on success without frequency changed.
+ *  - Negative on error.
+ */
+int power_cppc_cpufreq_freq_min(unsigned int lcore_id);
+
+/**
+ * Get the turbo status of a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 1 Turbo Boost is enabled on this lcore.
+ *  - 0 Turbo Boost is disabled on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_turbo_status(unsigned int lcore_id);
+
+/**
+ * Enable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost is enabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_enable_turbo(unsigned int lcore_id);
+
+/**
+ * Disable Turbo Boost on a specific lcore.
+ * It should be protected outside of this function for threadsafe.
+ *
+ * @param lcore_id
+ *  lcore id.
+ *
+ * @return
+ *  - 0 Turbo Boost disabled successfully on this lcore.
+ *  - Negative on error.
+ */
+int power_cppc_disable_turbo(unsigned int lcore_id);
+
+/**
+ * Returns power capabilities for a specific lcore.
+ *
+ * @param lcore_id
+ *  lcore id.
+ * @param caps
+ *  pointer to rte_power_core_capabilities object.
+ *
+ * @return
+ *  - 0 on success.
+ *  - Negative on error.
+ */
+int power_cppc_get_capabilities(unsigned int lcore_id,
+		struct rte_power_core_capabilities *caps);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _POWER_CPPC_CPUFREQ_H */
diff --git a/lib/power/rte_power.c b/lib/power/rte_power.c
index 98eaba9154..3cba56bac9 100644
--- a/lib/power/rte_power.c
+++ b/lib/power/rte_power.c
@@ -7,6 +7,7 @@
 
 #include "rte_power.h"
 #include "power_acpi_cpufreq.h"
+#include "power_cppc_cpufreq.h"
 #include "power_kvm_vm.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
@@ -54,6 +55,8 @@ rte_power_check_env_supported(enum power_management_env env)
 		return power_pstate_cpufreq_check_supported();
 	case PM_ENV_KVM_VM:
 		return power_kvm_vm_check_supported();
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_check_supported();
 	default:
 		rte_errno = EINVAL;
 		return -1;
@@ -110,6 +113,18 @@ rte_power_set_env(enum power_management_env env)
 		rte_power_freq_disable_turbo = power_pstate_disable_turbo;
 		rte_power_get_capabilities = power_pstate_get_capabilities;
 
+	} else if (env == PM_ENV_CPPC_CPUFREQ) {
+		rte_power_freqs = power_cppc_cpufreq_freqs;
+		rte_power_get_freq = power_cppc_cpufreq_get_freq;
+		rte_power_set_freq = power_cppc_cpufreq_set_freq;
+		rte_power_freq_up = power_cppc_cpufreq_freq_up;
+		rte_power_freq_down = power_cppc_cpufreq_freq_down;
+		rte_power_freq_min = power_cppc_cpufreq_freq_min;
+		rte_power_freq_max = power_cppc_cpufreq_freq_max;
+		rte_power_turbo_status = power_cppc_turbo_status;
+		rte_power_freq_enable_turbo = power_cppc_enable_turbo;
+		rte_power_freq_disable_turbo = power_cppc_disable_turbo;
+		rte_power_get_capabilities = power_cppc_get_capabilities;
 	} else {
 		RTE_LOG(ERR, POWER, "Invalid Power Management Environment(%d) set\n",
 				env);
@@ -153,6 +168,8 @@ rte_power_init(unsigned int lcore_id)
 		return power_kvm_vm_init(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_init(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_init(lcore_id);
 	default:
 		RTE_LOG(INFO, POWER, "Env isn't set yet!\n");
 	}
@@ -172,6 +189,13 @@ rte_power_init(unsigned int lcore_id)
 		goto out;
 	}
 
+	RTE_LOG(INFO, POWER, "Attempting to initialise CPPC power management...\n");
+	ret = power_cppc_cpufreq_init(lcore_id);
+	if (ret == 0) {
+		rte_power_set_env(PM_ENV_CPPC_CPUFREQ);
+		goto out;
+	}
+
 	RTE_LOG(INFO, POWER, "Attempting to initialise VM power management...\n");
 	ret = power_kvm_vm_init(lcore_id);
 	if (ret == 0) {
@@ -194,6 +218,8 @@ rte_power_exit(unsigned int lcore_id)
 		return power_kvm_vm_exit(lcore_id);
 	case PM_ENV_PSTATE_CPUFREQ:
 		return power_pstate_cpufreq_exit(lcore_id);
+	case PM_ENV_CPPC_CPUFREQ:
+		return power_cppc_cpufreq_exit(lcore_id);
 	default:
 		RTE_LOG(ERR, POWER, "Environment has not been set, unable to exit gracefully\n");
 
diff --git a/lib/power/rte_power.h b/lib/power/rte_power.h
index c8086bf6ba..c5759afa39 100644
--- a/lib/power/rte_power.h
+++ b/lib/power/rte_power.h
@@ -22,7 +22,7 @@ extern "C" {
 
 /* Power Management Environment State */
 enum power_management_env {PM_ENV_NOT_SET, PM_ENV_ACPI_CPUFREQ, PM_ENV_KVM_VM,
-		PM_ENV_PSTATE_CPUFREQ};
+		PM_ENV_PSTATE_CPUFREQ, PM_ENV_CPPC_CPUFREQ};
 
 /**
  * @warning
-- 
2.20.1

[dpdk-dev] [PATCH v8 2/2] test/power: round cpuinfo cur freq only in CPPC cpufreq

[Richael Zhuang]

On arm platform, the value in "/sys/.../cpuinfo_cur_freq" may not
be exactly the same as what was set when using CPPC cpufreq driver.
For other cpufreq driver, no need to round it currently, or else
this check will fail with turbo enabled. For example, with acpi_cpufreq,
cpuinfo_cur_freq can be 2401000 which is equal to freqs[0].It should
not be rounded to 2400000.

Fixes: 606a234c6d360 ("test/power: round CPU frequency to check")
Cc: richael.zhuang@arm.com
Cc: stable@dpdk.org

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
Acked-by: David Hunt <david.hunt@intel.com>
---
 app/test/test_power_cpufreq.c | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index 8516df4ca6..b8fc53925c 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -55,7 +55,9 @@ check_cur_freq(unsigned int lcore_id, uint32_t idx, bool turbo)
 	FILE *f;
 	char fullpath[PATH_MAX];
 	char buf[BUFSIZ];
+	enum power_management_env env;
 	uint32_t cur_freq;
+	uint32_t freq_conv;
 	int ret = -1;
 	int i;
 
@@ -80,15 +82,18 @@ check_cur_freq(unsigned int lcore_id, uint32_t idx, bool turbo)
 			goto fail_all;
 
 		cur_freq = strtoul(buf, NULL, TEST_POWER_CONVERT_TO_DECIMAL);
-
-		/* convert the frequency to nearest 100000 value
-		 * Ex: if cur_freq=1396789 then freq_conv=1400000
-		 * Ex: if cur_freq=800030 then freq_conv=800000
-		 */
-		unsigned int freq_conv = 0;
-		freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
-					/ TEST_ROUND_FREQ_TO_N_100000;
-		freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		freq_conv = cur_freq;
+
+		env = rte_power_get_env();
+		if (env == PM_ENV_CPPC_CPUFREQ) {
+			/* convert the frequency to nearest 100000 value
+			 * Ex: if cur_freq=1396789 then freq_conv=1400000
+			 * Ex: if cur_freq=800030 then freq_conv=800000
+			 */
+			freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
+						/ TEST_ROUND_FREQ_TO_N_100000;
+			freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		}
 
 		if (turbo)
 			ret = (freqs[idx] <= freq_conv ? 0 : -1);
-- 
2.20.1

Re: [dpdk-dev] [PATCH v8 0/2] power: add support for cppc cpufreq driver

[David Marchand]

On Fri, Jul 9, 2021 at 12:56 PM Richael Zhuang <richael.zhuang@arm.com> wrote:
>
> v8:
> fix build issue with clang

This cover letter is empty and we only have a partial changelog.
I'll trust David Hunt reviews and go ahead with the series.
Please put some effort in the cover letter and tracking changes for
future contributions.


>
> Richael Zhuang (2):
>   power: add support for cppc cpufreq
>   test/power: round cpuinfo cur freq only in CPPC cpufreq
>
>  app/test/test_power.c                  |   3 +-
>  app/test/test_power_cpufreq.c          |  26 +-
>  doc/guides/rel_notes/release_21_08.rst |   4 +
>  lib/power/meson.build                  |   1 +
>  lib/power/power_cppc_cpufreq.c         | 680 +++++++++++++++++++++++++
>  lib/power/power_cppc_cpufreq.h         | 230 +++++++++
>  lib/power/rte_power.c                  |  26 +
>  lib/power/rte_power.h                  |   2 +-
>  8 files changed, 960 insertions(+), 12 deletions(-)
>  create mode 100644 lib/power/power_cppc_cpufreq.c
>  create mode 100644 lib/power/power_cppc_cpufreq.h

Series applied, thanks.


-- 
David Marchand

Re: [dpdk-dev] [PATCH v8 0/2] power: add support for cppc cpufreq driver

[Richael Zhuang]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Friday, July 9, 2021 10:07 PM
> To: Richael Zhuang <Richael.Zhuang@arm.com>
> Cc: dev <dev@dpdk.org>; Yu Jiang <yux.jiang@intel.com>; David Hunt
> <david.hunt@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v8 0/2] power: add support for cppc cpufreq
> driver
> 
> On Fri, Jul 9, 2021 at 12:56 PM Richael Zhuang <richael.zhuang@arm.com>
> wrote:
> >
> > v8:
> > fix build issue with clang
> 
> This cover letter is empty and we only have a partial changelog.
> I'll trust David Hunt reviews and go ahead with the series.
> Please put some effort in the cover letter and tracking changes for future
> contributions.
>
Thanks for reminding! 
> 
> >
> > Richael Zhuang (2):
> >   power: add support for cppc cpufreq
> >   test/power: round cpuinfo cur freq only in CPPC cpufreq
> >
> >  app/test/test_power.c                  |   3 +-
> >  app/test/test_power_cpufreq.c          |  26 +-
> >  doc/guides/rel_notes/release_21_08.rst |   4 +
> >  lib/power/meson.build                  |   1 +
> >  lib/power/power_cppc_cpufreq.c         | 680 +++++++++++++++++++++++++
> >  lib/power/power_cppc_cpufreq.h         | 230 +++++++++
> >  lib/power/rte_power.c                  |  26 +
> >  lib/power/rte_power.h                  |   2 +-
> >  8 files changed, 960 insertions(+), 12 deletions(-)  create mode
> > 100644 lib/power/power_cppc_cpufreq.c  create mode 100644
> > lib/power/power_cppc_cpufreq.h
> 
> Series applied, thanks.
> 
> 
> --
> David Marchand

[dpdk-dev] [PATCH v7 2/2] test/power: round cpuinfo cur freq only in CPPC cpufreq

[Richael Zhuang]

On arm platform, the value in "/sys/.../cpuinfo_cur_freq" may not
be exactly the same as what was set when using CPPC cpufreq driver.
For other cpufreq driver, no need to round it currently, or else
this check will fail with turbo enabled. For example, with acpi_cpufreq,
cpuinfo_cur_freq can be 2401000 which is equal to freqs[0].It should
not be rounded to 2400000.

Fixes: 606a234c6d360 ("test/power: round CPU frequency to check")
Cc: richael.zhuang@arm.com
Cc: stable@dpdk.org

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
---
 app/test/test_power_cpufreq.c | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index 8516df4ca6..b8fc53925c 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -55,7 +55,9 @@ check_cur_freq(unsigned int lcore_id, uint32_t idx, bool turbo)
 	FILE *f;
 	char fullpath[PATH_MAX];
 	char buf[BUFSIZ];
+	enum power_management_env env;
 	uint32_t cur_freq;
+	uint32_t freq_conv;
 	int ret = -1;
 	int i;
 
@@ -80,15 +82,18 @@ check_cur_freq(unsigned int lcore_id, uint32_t idx, bool turbo)
 			goto fail_all;
 
 		cur_freq = strtoul(buf, NULL, TEST_POWER_CONVERT_TO_DECIMAL);
-
-		/* convert the frequency to nearest 100000 value
-		 * Ex: if cur_freq=1396789 then freq_conv=1400000
-		 * Ex: if cur_freq=800030 then freq_conv=800000
-		 */
-		unsigned int freq_conv = 0;
-		freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
-					/ TEST_ROUND_FREQ_TO_N_100000;
-		freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		freq_conv = cur_freq;
+
+		env = rte_power_get_env();
+		if (env == PM_ENV_CPPC_CPUFREQ) {
+			/* convert the frequency to nearest 100000 value
+			 * Ex: if cur_freq=1396789 then freq_conv=1400000
+			 * Ex: if cur_freq=800030 then freq_conv=800000
+			 */
+			freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
+						/ TEST_ROUND_FREQ_TO_N_100000;
+			freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		}
 
 		if (turbo)
 			ret = (freqs[idx] <= freq_conv ? 0 : -1);
-- 
2.20.1

[dpdk-dev] [PATCH v6 2/2] test/power: round cpuinfo cur freq only in CPPC cpufreq

[Richael Zhuang]

On arm platform, the value in "/sys/.../cpuinfo_cur_freq" may not
be exactly the same as what was set when using CPPC cpufreq driver.
For other cpufreq driver, no need to round it currently, or else
this check will fail with turbo enabled. For example, with acpi_cpufreq,
cpuinfo_cur_freq can be 2401000 which is equal to freqs[0].It should
not be rounded to 2400000.

Fixes: 606a234c6d360 ("test/power: round CPU frequency to check")
Cc: richael.zhuang@arm.com
Cc: stable@dpdk.org

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
---
 app/test/test_power_cpufreq.c | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index 8516df4ca..b8fc53925 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -55,7 +55,9 @@ check_cur_freq(unsigned int lcore_id, uint32_t idx, bool turbo)
 	FILE *f;
 	char fullpath[PATH_MAX];
 	char buf[BUFSIZ];
+	enum power_management_env env;
 	uint32_t cur_freq;
+	uint32_t freq_conv;
 	int ret = -1;
 	int i;
 
@@ -80,15 +82,18 @@ check_cur_freq(unsigned int lcore_id, uint32_t idx, bool turbo)
 			goto fail_all;
 
 		cur_freq = strtoul(buf, NULL, TEST_POWER_CONVERT_TO_DECIMAL);
-
-		/* convert the frequency to nearest 100000 value
-		 * Ex: if cur_freq=1396789 then freq_conv=1400000
-		 * Ex: if cur_freq=800030 then freq_conv=800000
-		 */
-		unsigned int freq_conv = 0;
-		freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
-					/ TEST_ROUND_FREQ_TO_N_100000;
-		freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		freq_conv = cur_freq;
+
+		env = rte_power_get_env();
+		if (env == PM_ENV_CPPC_CPUFREQ) {
+			/* convert the frequency to nearest 100000 value
+			 * Ex: if cur_freq=1396789 then freq_conv=1400000
+			 * Ex: if cur_freq=800030 then freq_conv=800000
+			 */
+			freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
+						/ TEST_ROUND_FREQ_TO_N_100000;
+			freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		}
 
 		if (turbo)
 			ret = (freqs[idx] <= freq_conv ? 0 : -1);
-- 
2.20.1

Re: [dpdk-dev] [PATCH v6 2/2] test/power: round cpuinfo cur freq only in CPPC cpufreq

[David Hunt]

On 8/7/2021 3:34 AM, Richael Zhuang wrote:
> On arm platform, the value in "/sys/.../cpuinfo_cur_freq" may not
> be exactly the same as what was set when using CPPC cpufreq driver.
> For other cpufreq driver, no need to round it currently, or else
> this check will fail with turbo enabled. For example, with acpi_cpufreq,
> cpuinfo_cur_freq can be 2401000 which is equal to freqs[0].It should
> not be rounded to 2400000.
>
> Fixes: 606a234c6d360 ("test/power: round CPU frequency to check")
> Cc: richael.zhuang@arm.com
> Cc: stable@dpdk.org
>
> Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> ---
>   app/test/test_power_cpufreq.c | 23 ++++++++++++++---------
>   1 file changed, 14 insertions(+), 9 deletions(-)
>

Acked-by: David Hunt <david.hunt@intel.com>

[dpdk-dev] [PATCH v5 2/2] test/power: round cpuinfo cur freq only when using CPPC cpufreq

[Richael Zhuang]

On arm platform, the value in "/sys/.../cpuinfo_cur_freq" may not
be exactly the same as what was set when using CPPC cpufreq driver.
For other cpufreq driver, no need to round it currently, or else
this check will fail with turbo enabled. For example, with acpi_cpufreq,
cpuinfo_cur_freq can be 2401000 which is equal to freqs[0].It should
not be rounded to 2400000.

Fixes: 606a234c6d360 ("test/power: round CPU frequency to check")
Cc: richael.zhuang@arm.com
Cc: stable@dpdk.org

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
---
 app/test/test_power_cpufreq.c | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index 8516df4ca..b8fc53925 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -55,7 +55,9 @@ check_cur_freq(unsigned int lcore_id, uint32_t idx, bool turbo)
 	FILE *f;
 	char fullpath[PATH_MAX];
 	char buf[BUFSIZ];
+	enum power_management_env env;
 	uint32_t cur_freq;
+	uint32_t freq_conv;
 	int ret = -1;
 	int i;
 
@@ -80,15 +82,18 @@ check_cur_freq(unsigned int lcore_id, uint32_t idx, bool turbo)
 			goto fail_all;
 
 		cur_freq = strtoul(buf, NULL, TEST_POWER_CONVERT_TO_DECIMAL);
-
-		/* convert the frequency to nearest 100000 value
-		 * Ex: if cur_freq=1396789 then freq_conv=1400000
-		 * Ex: if cur_freq=800030 then freq_conv=800000
-		 */
-		unsigned int freq_conv = 0;
-		freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
-					/ TEST_ROUND_FREQ_TO_N_100000;
-		freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		freq_conv = cur_freq;
+
+		env = rte_power_get_env();
+		if (env == PM_ENV_CPPC_CPUFREQ) {
+			/* convert the frequency to nearest 100000 value
+			 * Ex: if cur_freq=1396789 then freq_conv=1400000
+			 * Ex: if cur_freq=800030 then freq_conv=800000
+			 */
+			freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
+						/ TEST_ROUND_FREQ_TO_N_100000;
+			freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		}
 
 		if (turbo)
 			ret = (freqs[idx] <= freq_conv ? 0 : -1);
-- 
2.20.1

[dpdk-dev] [PATCH v4 2/2] test/power: round cpuinfo cur freq only when using CPPC cpufreq

[Richael Zhuang]

On arm platform, the value in "/sys/.../cpuinfo_cur_freq" may not
be exactly the same as what was set when using CPPC cpufreq driver.
For other cpufreq driver, no need to round it currently, or else
this check will fail with turbo enabled. For example, with acpi_cpufreq,
cpuinfo_cur_freq can be 2401000 which is equal to freqs[0].It should
not be rounded to 2400000.

Fixes: 606a234c6d360 ("test/power: round CPU frequency to check")
Cc: richael.zhuang@arm.com
Cc: stable@dpdk.org

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
---
 app/test/test_power_cpufreq.c | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index 8516df4ca..b8fc53925 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -55,7 +55,9 @@ check_cur_freq(unsigned int lcore_id, uint32_t idx, bool turbo)
 	FILE *f;
 	char fullpath[PATH_MAX];
 	char buf[BUFSIZ];
+	enum power_management_env env;
 	uint32_t cur_freq;
+	uint32_t freq_conv;
 	int ret = -1;
 	int i;
 
@@ -80,15 +82,18 @@ check_cur_freq(unsigned int lcore_id, uint32_t idx, bool turbo)
 			goto fail_all;
 
 		cur_freq = strtoul(buf, NULL, TEST_POWER_CONVERT_TO_DECIMAL);
-
-		/* convert the frequency to nearest 100000 value
-		 * Ex: if cur_freq=1396789 then freq_conv=1400000
-		 * Ex: if cur_freq=800030 then freq_conv=800000
-		 */
-		unsigned int freq_conv = 0;
-		freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
-					/ TEST_ROUND_FREQ_TO_N_100000;
-		freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		freq_conv = cur_freq;
+
+		env = rte_power_get_env();
+		if (env == PM_ENV_CPPC_CPUFREQ) {
+			/* convert the frequency to nearest 100000 value
+			 * Ex: if cur_freq=1396789 then freq_conv=1400000
+			 * Ex: if cur_freq=800030 then freq_conv=800000
+			 */
+			freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
+						/ TEST_ROUND_FREQ_TO_N_100000;
+			freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		}
 
 		if (turbo)
 			ret = (freqs[idx] <= freq_conv ? 0 : -1);
-- 
2.20.1

Re: [dpdk-dev] [PATCH v4 2/2] test/power: round cpuinfo cur freq only when using CPPC cpufreq

[David Hunt]

Hi Richael,

On 23/6/2021 4:55 AM, Richael Zhuang wrote:
> On arm platform, the value in "/sys/.../cpuinfo_cur_freq" may not
> be exactly the same as what was set when using CPPC cpufreq driver.
> For other cpufreq driver, no need to round it currently, or else
> this check will fail with turbo enabled. For example, with acpi_cpufreq,
> cpuinfo_cur_freq can be 2401000 which is equal to freqs[0].It should
> not be rounded to 2400000.
>
> Fixes: 606a234c6d360 ("test/power: round CPU frequency to check")
> Cc: richael.zhuang@arm.com
> Cc: stable@dpdk.org
>
> Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
> ---
>   app/test/test_power_cpufreq.c | 23 ++++++++++++++---------
>   1 file changed, 14 insertions(+), 9 deletions(-)

--snip--

Makes sense.

Reviewed-by: David Hunt <david.hunt@intel.com>

[dpdk-dev] [PATCH v3 2/2] test/power: round cpuinfo cur freq only when using CPPC cpufreq

[Richael Zhuang]

On arm platform, the value in "/sys/.../cpuinfo_cur_freq" may not
be exactly the same as what was set when using CPPC cpufreq driver.
For other cpufreq driver, no need to round it currently, or else
this check will fail with turbo enabled. For example, with acpi_cpufreq,
cpuinfo_cur_freq can be 2401000 which is equal to freqs[0].It should
not be rounded to 2400000.

Fixes: 606a234c6d360 ("test/power: round CPU frequency to check")
Cc: richael.zhuang@arm.com
Cc: stable@dpdk.org

Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
---
 app/test/test_power_cpufreq.c | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/app/test/test_power_cpufreq.c b/app/test/test_power_cpufreq.c
index 87d07191d..0e195afc9 100644
--- a/app/test/test_power_cpufreq.c
+++ b/app/test/test_power_cpufreq.c
@@ -53,7 +53,9 @@ check_cur_freq(unsigned lcore_id, uint32_t idx)
 	FILE *f;
 	char fullpath[PATH_MAX];
 	char buf[BUFSIZ];
+	enum power_management_env env;
 	uint32_t cur_freq;
+	uint32_t freq_conv;
 	int ret = -1;
 	int i;
 
@@ -71,15 +73,18 @@ check_cur_freq(unsigned lcore_id, uint32_t idx)
 			goto fail_all;
 
 		cur_freq = strtoul(buf, NULL, TEST_POWER_CONVERT_TO_DECIMAL);
-
-		/* convert the frequency to nearest 100000 value
-		 * Ex: if cur_freq=1396789 then freq_conv=1400000
-		 * Ex: if cur_freq=800030 then freq_conv=800000
-		 */
-		unsigned int freq_conv = 0;
-		freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
-					/ TEST_ROUND_FREQ_TO_N_100000;
-		freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		freq_conv = cur_freq;
+
+		env = rte_power_get_env();
+		if (env == PM_ENV_CPPC_CPUFREQ) {
+			/* convert the frequency to nearest 100000 value
+			 * Ex: if cur_freq=1396789 then freq_conv=1400000
+			 * Ex: if cur_freq=800030 then freq_conv=800000
+			 */
+			freq_conv = (cur_freq + TEST_FREQ_ROUNDING_DELTA)
+						/ TEST_ROUND_FREQ_TO_N_100000;
+			freq_conv = freq_conv * TEST_ROUND_FREQ_TO_N_100000;
+		}
 
 		ret = (freqs[idx] == freq_conv ? 0 : -1);
 
-- 
2.20.1