[PATCH 0/4] Add APIs for configurable power options

[PATCH 0/4] Add APIs for configurable power options

[Kevin Laatz]

The power library contains some variables which are currently set by
defines, hard-coded values or set using sysfs values. In order to
configure these, code changes and recompiles are required, making
configuring these variables tedious.

This patchset introduces some new get/set APIs which allow users and
applications to configure there settings to suit their use-cases.
In addition, CLI options have been added to l3fwd_power to demonstrate
how an application could use these APIs to expose the options to users
without needing code changes to configure them.

Kevin Laatz (4):
  lib/power: add get and set API for emptypoll max
  lib/power: add get and set API for pause duration
  lib/power: add get and set API for scaling freq min and max with
    pstate mode
  examples/l3fwd_power: add cli for configurable options

 examples/l3fwd-power/main.c      |  75 ++++++++++++++++-
 lib/power/power_pstate_cpufreq.c |  22 +++--
 lib/power/rte_power_pmd_mgmt.c   | 112 +++++++++++++++++++++++--
 lib/power/rte_power_pmd_mgmt.h   | 138 +++++++++++++++++++++++++++++++
 lib/power/version.map            |  10 +++
 5 files changed, 346 insertions(+), 11 deletions(-)

-- 
2.31.1

[PATCH 1/4] lib/power: add get and set API for emptypoll max

[Kevin Laatz]

Add new get/set APIs to configure emptypoll max which is used to
determine when a queue can go into sleep state.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
---
 lib/power/rte_power_pmd_mgmt.c | 21 ++++++++++++++++++---
 lib/power/rte_power_pmd_mgmt.h | 27 +++++++++++++++++++++++++++
 lib/power/version.map          |  4 ++++
 3 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index 39a2b4cd23..dfb7ca9187 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -11,7 +11,7 @@
 
 #include "rte_power_pmd_mgmt.h"
 
-#define EMPTYPOLL_MAX  512
+unsigned int emptypoll_max;
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -206,7 +206,7 @@ queue_can_sleep(struct pmd_core_cfg *cfg, struct queue_list_entry *qcfg)
 	qcfg->n_empty_polls++;
 
 	/* if we haven't reached threshold for empty polls, we can't sleep */
-	if (qcfg->n_empty_polls <= EMPTYPOLL_MAX)
+	if (qcfg->n_empty_polls <= emptypoll_max)
 		return false;
 
 	/*
@@ -290,7 +290,7 @@ clb_umwait(uint16_t port_id, uint16_t qidx, struct rte_mbuf **pkts __rte_unused,
 	/* this callback can't do more than one queue, omit multiqueue logic */
 	if (unlikely(nb_rx == 0)) {
 		queue_conf->n_empty_polls++;
-		if (unlikely(queue_conf->n_empty_polls > EMPTYPOLL_MAX)) {
+		if (unlikely(queue_conf->n_empty_polls > emptypoll_max)) {
 			struct rte_power_monitor_cond pmc;
 			int ret;
 
@@ -661,6 +661,18 @@ rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
 	return 0;
 }
 
+void
+rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max)
+{
+	emptypoll_max = max;
+}
+
+unsigned int
+rte_power_pmd_mgmt_get_emptypoll_max(void)
+{
+	return emptypoll_max;
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
 
@@ -669,4 +681,7 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 		struct pmd_core_cfg *cfg = &lcore_cfgs[i];
 		TAILQ_INIT(&cfg->head);
 	}
+
+	/* initialize config defaults */
+	emptypoll_max = 512;
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index 444e7b8a66..d5a94f8187 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -90,6 +90,33 @@ int
 rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
 		uint16_t port_id, uint16_t queue_id);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set a emptypoll_max to specified value. Used to specify the number of empty
+ * polls to wait before entering sleep state.
+ *
+ * @param max
+ *   The value to set emptypoll_max to.
+ */
+__rte_experimental
+void
+rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current value of emptypoll_max.
+ *
+ * @return
+ *   The current emptypoll_max value
+ */
+__rte_experimental
+unsigned int
+rte_power_pmd_mgmt_get_emptypoll_max(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 6ec6d5d96d..8bcd497e06 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -38,4 +38,8 @@ EXPERIMENTAL {
 	# added in 21.02
 	rte_power_ethdev_pmgmt_queue_disable;
 	rte_power_ethdev_pmgmt_queue_enable;
+
+	# added in 22.07
+	rte_power_pmd_mgmt_set_emptypoll_max;
+	rte_power_pmd_mgmt_get_emptypoll_max;
 };
-- 
2.31.1

Re: [PATCH 1/4] lib/power: add get and set API for emptypoll max

[Ray Kinsella]

Kevin Laatz <kevin.laatz@intel.com> writes:

> Add new get/set APIs to configure emptypoll max which is used to
> determine when a queue can go into sleep state.
>
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> ---
>  lib/power/rte_power_pmd_mgmt.c | 21 ++++++++++++++++++---
>  lib/power/rte_power_pmd_mgmt.h | 27 +++++++++++++++++++++++++++
>  lib/power/version.map          |  4 ++++
>  3 files changed, 49 insertions(+), 3 deletions(-)
>
> diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
> index 39a2b4cd23..dfb7ca9187 100644
> --- a/lib/power/rte_power_pmd_mgmt.c
> +++ b/lib/power/rte_power_pmd_mgmt.c
> @@ -11,7 +11,7 @@
>  
>  #include "rte_power_pmd_mgmt.h"
>  
> -#define EMPTYPOLL_MAX  512
> +unsigned int emptypoll_max;
>  
>  /* store some internal state */
>  static struct pmd_conf_data {
> @@ -206,7 +206,7 @@ queue_can_sleep(struct pmd_core_cfg *cfg, struct queue_list_entry *qcfg)
>  	qcfg->n_empty_polls++;
>  
>  	/* if we haven't reached threshold for empty polls, we can't sleep */
> -	if (qcfg->n_empty_polls <= EMPTYPOLL_MAX)
> +	if (qcfg->n_empty_polls <= emptypoll_max)
>  		return false;
>  
>  	/*
> @@ -290,7 +290,7 @@ clb_umwait(uint16_t port_id, uint16_t qidx, struct rte_mbuf **pkts __rte_unused,
>  	/* this callback can't do more than one queue, omit multiqueue logic */
>  	if (unlikely(nb_rx == 0)) {
>  		queue_conf->n_empty_polls++;
> -		if (unlikely(queue_conf->n_empty_polls > EMPTYPOLL_MAX)) {
> +		if (unlikely(queue_conf->n_empty_polls > emptypoll_max)) {
>  			struct rte_power_monitor_cond pmc;
>  			int ret;
>  
> @@ -661,6 +661,18 @@ rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
>  	return 0;
>  }
>  
> +void
> +rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max)
> +{
> +	emptypoll_max = max;
> +}
> +
> +unsigned int
> +rte_power_pmd_mgmt_get_emptypoll_max(void)
> +{
> +	return emptypoll_max;
> +}
> +
>  RTE_INIT(rte_power_ethdev_pmgmt_init) {
>  	size_t i;
>  
> @@ -669,4 +681,7 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
>  		struct pmd_core_cfg *cfg = &lcore_cfgs[i];
>  		TAILQ_INIT(&cfg->head);
>  	}
> +
> +	/* initialize config defaults */
> +	emptypoll_max = 512;
>  }
> diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
> index 444e7b8a66..d5a94f8187 100644
> --- a/lib/power/rte_power_pmd_mgmt.h
> +++ b/lib/power/rte_power_pmd_mgmt.h
> @@ -90,6 +90,33 @@ int
>  rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
>  		uint16_t port_id, uint16_t queue_id);
>  
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
> + *
> + * Set a emptypoll_max to specified value. Used to specify the number of empty
> + * polls to wait before entering sleep state.
> + *
> + * @param max
> + *   The value to set emptypoll_max to.
> + */
> +__rte_experimental
> +void
> +rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
> + *
> + * Get the current value of emptypoll_max.
> + *
> + * @return
> + *   The current emptypoll_max value
> + */
> +__rte_experimental
> +unsigned int
> +rte_power_pmd_mgmt_get_emptypoll_max(void);
> +
>  #ifdef __cplusplus
>  }
>  #endif
> diff --git a/lib/power/version.map b/lib/power/version.map
> index 6ec6d5d96d..8bcd497e06 100644
> --- a/lib/power/version.map
> +++ b/lib/power/version.map
> @@ -38,4 +38,8 @@ EXPERIMENTAL {
>  	# added in 21.02
>  	rte_power_ethdev_pmgmt_queue_disable;
>  	rte_power_ethdev_pmgmt_queue_enable;
> +
> +	# added in 22.07
> +	rte_power_pmd_mgmt_set_emptypoll_max;
> +	rte_power_pmd_mgmt_get_emptypoll_max;

minor niggle, get then set. 

>  };


-- 
Regards, Ray K

Re: [PATCH 1/4] lib/power: add get and set API for emptypoll max

[Kevin Laatz]

On 15/04/2022 15:43, Ray Kinsella wrote:
> Kevin Laatz <kevin.laatz@intel.com> writes:
>
>> Add new get/set APIs to configure emptypoll max which is used to
>> determine when a queue can go into sleep state.
>>
>> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
>> ---
>>   lib/power/rte_power_pmd_mgmt.c | 21 ++++++++++++++++++---
>>   lib/power/rte_power_pmd_mgmt.h | 27 +++++++++++++++++++++++++++
>>   lib/power/version.map          |  4 ++++
>>   3 files changed, 49 insertions(+), 3 deletions(-)
>>
>> diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
>> index 39a2b4cd23..dfb7ca9187 100644
>> --- a/lib/power/rte_power_pmd_mgmt.c
>> +++ b/lib/power/rte_power_pmd_mgmt.c
>> @@ -11,7 +11,7 @@
>>   
>>   #include "rte_power_pmd_mgmt.h"
>>   
>> -#define EMPTYPOLL_MAX  512
>> +unsigned int emptypoll_max;
>>   
>>   /* store some internal state */
>>   static struct pmd_conf_data {
>> @@ -206,7 +206,7 @@ queue_can_sleep(struct pmd_core_cfg *cfg, struct queue_list_entry *qcfg)
>>   	qcfg->n_empty_polls++;
>>   
>>   	/* if we haven't reached threshold for empty polls, we can't sleep */
>> -	if (qcfg->n_empty_polls <= EMPTYPOLL_MAX)
>> +	if (qcfg->n_empty_polls <= emptypoll_max)
>>   		return false;
>>   
>>   	/*
>> @@ -290,7 +290,7 @@ clb_umwait(uint16_t port_id, uint16_t qidx, struct rte_mbuf **pkts __rte_unused,
>>   	/* this callback can't do more than one queue, omit multiqueue logic */
>>   	if (unlikely(nb_rx == 0)) {
>>   		queue_conf->n_empty_polls++;
>> -		if (unlikely(queue_conf->n_empty_polls > EMPTYPOLL_MAX)) {
>> +		if (unlikely(queue_conf->n_empty_polls > emptypoll_max)) {
>>   			struct rte_power_monitor_cond pmc;
>>   			int ret;
>>   
>> @@ -661,6 +661,18 @@ rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
>>   	return 0;
>>   }
>>   
>> +void
>> +rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max)
>> +{
>> +	emptypoll_max = max;
>> +}
>> +
>> +unsigned int
>> +rte_power_pmd_mgmt_get_emptypoll_max(void)
>> +{
>> +	return emptypoll_max;
>> +}
>> +
>>   RTE_INIT(rte_power_ethdev_pmgmt_init) {
>>   	size_t i;
>>   
>> @@ -669,4 +681,7 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
>>   		struct pmd_core_cfg *cfg = &lcore_cfgs[i];
>>   		TAILQ_INIT(&cfg->head);
>>   	}
>> +
>> +	/* initialize config defaults */
>> +	emptypoll_max = 512;
>>   }
>> diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
>> index 444e7b8a66..d5a94f8187 100644
>> --- a/lib/power/rte_power_pmd_mgmt.h
>> +++ b/lib/power/rte_power_pmd_mgmt.h
>> @@ -90,6 +90,33 @@ int
>>   rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
>>   		uint16_t port_id, uint16_t queue_id);
>>   
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
>> + *
>> + * Set a emptypoll_max to specified value. Used to specify the number of empty
>> + * polls to wait before entering sleep state.
>> + *
>> + * @param max
>> + *   The value to set emptypoll_max to.
>> + */
>> +__rte_experimental
>> +void
>> +rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max);
>> +
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
>> + *
>> + * Get the current value of emptypoll_max.
>> + *
>> + * @return
>> + *   The current emptypoll_max value
>> + */
>> +__rte_experimental
>> +unsigned int
>> +rte_power_pmd_mgmt_get_emptypoll_max(void);
>> +
>>   #ifdef __cplusplus
>>   }
>>   #endif
>> diff --git a/lib/power/version.map b/lib/power/version.map
>> index 6ec6d5d96d..8bcd497e06 100644
>> --- a/lib/power/version.map
>> +++ b/lib/power/version.map
>> @@ -38,4 +38,8 @@ EXPERIMENTAL {
>>   	# added in 21.02
>>   	rte_power_ethdev_pmgmt_queue_disable;
>>   	rte_power_ethdev_pmgmt_queue_enable;
>> +
>> +	# added in 22.07
>> +	rte_power_pmd_mgmt_set_emptypoll_max;
>> +	rte_power_pmd_mgmt_get_emptypoll_max;
> minor niggle, get then set.
>
Re-ordered in v2, thanks.

[PATCH 2/4] lib/power: add get and set API for pause duration

[Kevin Laatz]

Add new get/set API for configuring 'pause_duration' which used to adjust
the pause mode callback duration.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
---
 lib/power/rte_power_pmd_mgmt.c | 26 ++++++++++++++++++++++++--
 lib/power/rte_power_pmd_mgmt.h | 31 +++++++++++++++++++++++++++++++
 lib/power/version.map          |  2 ++
 3 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index dfb7ca9187..ab92a93aa0 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -12,6 +12,7 @@
 #include "rte_power_pmd_mgmt.h"
 
 unsigned int emptypoll_max;
+unsigned int pause_duration;
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -315,6 +316,7 @@ clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,
 	struct queue_list_entry *queue_conf = arg;
 	struct pmd_core_cfg *lcore_conf;
 	const bool empty = nb_rx == 0;
+	uint32_t pause_duration = rte_power_pmd_mgmt_get_pause_duration();
 
 	lcore_conf = &lcore_cfgs[lcore];
 
@@ -334,11 +336,11 @@ clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,
 		if (global_data.intrinsics_support.power_pause) {
 			const uint64_t cur = rte_rdtsc();
 			const uint64_t wait_tsc =
-					cur + global_data.tsc_per_us;
+					cur + global_data.tsc_per_us * pause_duration;
 			rte_power_pause(wait_tsc);
 		} else {
 			uint64_t i;
-			for (i = 0; i < global_data.pause_per_us; i++)
+			for (i = 0; i < global_data.pause_per_us * pause_duration; i++)
 				rte_pause();
 		}
 	}
@@ -673,6 +675,25 @@ rte_power_pmd_mgmt_get_emptypoll_max(void)
 	return emptypoll_max;
 }
 
+int
+rte_power_pmd_mgmt_set_pause_duration(unsigned int duration)
+{
+	if (duration == 0) {
+		printf("Pause duration must be greater than 0, value unchanged\n");
+		rte_errno = EINVAL;
+		return -1;
+	}
+	pause_duration = duration;
+
+	return 0;
+}
+
+unsigned int
+rte_power_pmd_mgmt_get_pause_duration(void)
+{
+	return pause_duration;
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
 
@@ -684,4 +705,5 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 
 	/* initialize config defaults */
 	emptypoll_max = 512;
+	pause_duration = 1;
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index d5a94f8187..18a9c3abb5 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -117,6 +117,37 @@ __rte_experimental
 unsigned int
 rte_power_pmd_mgmt_get_emptypoll_max(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the pause_duration. Used to adjust the pause mode callback duration.
+ *
+ * @note Duration must be greater than zero.
+ *
+ * @param duration
+ *   The value to set pause_duration to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_pause_duration(unsigned int duration);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current value of pause_duration.
+ *
+ * @return
+ *   The current pause_duration value.
+ */
+__rte_experimental
+unsigned int
+rte_power_pmd_mgmt_get_pause_duration(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 8bcd497e06..5890e6e429 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -42,4 +42,6 @@ EXPERIMENTAL {
 	# added in 22.07
 	rte_power_pmd_mgmt_set_emptypoll_max;
 	rte_power_pmd_mgmt_get_emptypoll_max;
+	rte_power_pmd_mgmt_set_pause_duration;
+	rte_power_pmd_mgmt_get_pause_duration;
 };
-- 
2.31.1

[PATCH 3/4] lib/power: add get and set API for scaling freq min and max with pstate mode

[Kevin Laatz]

Add new get/set API to allow the user or application to set the minimum
and maximum frequencies to use when scaling.
Previously, the frequency range was determined by the HW capabilities of
the CPU. With this new API, the user or application can constrain this
if required.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
---
 lib/power/power_pstate_cpufreq.c | 22 +++++++--
 lib/power/rte_power_pmd_mgmt.c   | 65 ++++++++++++++++++++++++++
 lib/power/rte_power_pmd_mgmt.h   | 80 ++++++++++++++++++++++++++++++++
 lib/power/version.map            |  4 ++
 4 files changed, 166 insertions(+), 5 deletions(-)

diff --git a/lib/power/power_pstate_cpufreq.c b/lib/power/power_pstate_cpufreq.c
index f4c36179ec..db10deafe2 100644
--- a/lib/power/power_pstate_cpufreq.c
+++ b/lib/power/power_pstate_cpufreq.c
@@ -12,6 +12,7 @@
 
 #include <rte_memcpy.h>
 
+#include "rte_power_pmd_mgmt.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
 
@@ -354,6 +355,7 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	FILE *f_min = NULL, *f_max = NULL;
 	int ret = -1;
 	uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
+	int config_min_freq, config_max_freq;
 	uint32_t i, num_freqs = 0;
 
 	/* open all files */
@@ -388,6 +390,16 @@ power_get_available_freqs(struct pstate_power_info *pi)
 		goto out;
 	}
 
+	/* check for config set by user or application to limit frequency range */
+	config_min_freq = rte_power_pmd_mgmt_get_scaling_freq_min(pi->lcore_id);
+	if (config_min_freq < 0)
+		goto out;
+	config_max_freq = rte_power_pmd_mgmt_get_scaling_freq_max(pi->lcore_id);
+	if (config_max_freq < 0)
+		goto out;
+	sys_min_freq = RTE_MAX(sys_min_freq, (uint32_t)config_min_freq);
+	sys_max_freq = RTE_MIN(sys_max_freq, (uint32_t)config_max_freq);
+
 	if (sys_max_freq < sys_min_freq)
 		goto out;
 
@@ -411,8 +423,8 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	/* If turbo is available then there is one extra freq bucket
 	 * to store the sys max freq which value is base_max +1
 	 */
-	num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
-		pi->turbo_available;
+	num_freqs = (RTE_MIN(base_max_freq, sys_max_freq) - sys_min_freq) / BUS_FREQ
+			+ 1 + pi->turbo_available;
 	if (num_freqs >= RTE_MAX_LCORE_FREQS) {
 		RTE_LOG(ERR, POWER, "Too many available frequencies: %d\n",
 				num_freqs);
@@ -427,10 +439,10 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	 */
 	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 		if ((i == 0) && pi->turbo_available)
-			pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
+			pi->freqs[pi->nb_freqs++] = RTE_MIN(base_max_freq, sys_max_freq) + 1;
 		else
-			pi->freqs[pi->nb_freqs++] =
-			base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
+			pi->freqs[pi->nb_freqs++] = RTE_MIN(base_max_freq, sys_max_freq) -
+					(i - pi->turbo_available) * BUS_FREQ;
 	}
 
 	ret = 0;
diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index ab92a93aa0..5ba050dc95 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -10,9 +10,12 @@
 #include <rte_power_intrinsics.h>
 
 #include "rte_power_pmd_mgmt.h"
+#include "power_common.h"
 
 unsigned int emptypoll_max;
 unsigned int pause_duration;
+unsigned int scale_freq_min[RTE_MAX_LCORE];
+unsigned int scale_freq_max[RTE_MAX_LCORE];
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -694,8 +697,65 @@ rte_power_pmd_mgmt_get_pause_duration(void)
 	return pause_duration;
 }
 
+int
+rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		rte_errno = EINVAL;
+		return -1;
+	}
+	scale_freq_min[lcore] = min;
+
+	return 0;
+}
+
+int
+rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		rte_errno = EINVAL;
+		return -1;
+	}
+	scale_freq_max[lcore] = max;
+
+	return 0;
+}
+
+int
+rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		rte_errno = EINVAL;
+		return -1;
+	}
+
+	if (scale_freq_max[lcore] == 0)
+		RTE_LOG(DEBUG, POWER, "Scaling freq min config not set. Using sysfs min freq.\n");
+
+	return scale_freq_min[lcore];
+}
+
+int
+rte_power_pmd_mgmt_get_scaling_freq_max(unsigned int lcore)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		rte_errno = EINVAL;
+		return -1;
+	}
+
+	if (scale_freq_max[lcore] == UINT32_MAX)
+		RTE_LOG(DEBUG, POWER, "Scaling freq max config not set. Using sysfs max freq.\n");
+
+	return scale_freq_max[lcore];
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
+	int j;
 
 	/* initialize all tailqs */
 	for (i = 0; i < RTE_DIM(lcore_cfgs); i++) {
@@ -706,4 +766,9 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	/* initialize config defaults */
 	emptypoll_max = 512;
 	pause_duration = 1;
+	/* scaling defaults out of range to ensure not used unless set by user or app */
+	for (j = 0; j < RTE_MAX_LCORE; j++) {
+		scale_freq_min[j] = 0;
+		scale_freq_max[j] = UINT32_MAX;
+	}
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index 18a9c3abb5..74e3fa710b 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -148,6 +148,86 @@ __rte_experimental
 unsigned int
 rte_power_pmd_mgmt_get_pause_duration(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the min frequency to be used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to set the min frequency for.
+ * @param min
+ *   The value, in Hertz, to set the minimum frequency to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the max frequency to be used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to set the max frequency for.
+ * @param max
+ *   The value, in Hertz, to set the maximum frequency to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current configured min frequency used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to get the min frequency for.
+ * @return
+ *   0 if no value has been configured via the 'set' API.
+ *   >0 if a minimum frequency has been configured. Value is the minimum frequency
+ *   , in Hertz, used for frequency scaling.
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current configured max frequency used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to get the max frequency for.
+ * @return
+ *   UINT32_MAX if no value has been configured via the 'set' API.
+ *   On success, the current configured maximum frequency, in Hertz, used for
+ *   frequency scaling..
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_get_scaling_freq_max(unsigned int lcore);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 5890e6e429..2cf08b7c5a 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -44,4 +44,8 @@ EXPERIMENTAL {
 	rte_power_pmd_mgmt_get_emptypoll_max;
 	rte_power_pmd_mgmt_set_pause_duration;
 	rte_power_pmd_mgmt_get_pause_duration;
+	rte_power_pmd_mgmt_set_scaling_freq_min;
+	rte_power_pmd_mgmt_set_scaling_freq_max;
+	rte_power_pmd_mgmt_get_scaling_freq_min;
+	rte_power_pmd_mgmt_get_scaling_freq_max;
 };
-- 
2.31.1

[PATCH 4/4] examples/l3fwd_power: add cli for configurable options

[Kevin Laatz]

Add CLI options to l3fwd_power to utilize the new power APIs introduced in
this patchset. These CLI options allow the user to configure the
heuritstics made available through the new API via the l3fwd_power
application options.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
---
 examples/l3fwd-power/main.c | 75 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 74 insertions(+), 1 deletion(-)

diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 20e5b59af9..f480de2420 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -265,6 +265,10 @@ static struct rte_eth_conf port_conf = {
 };
 
 static uint32_t max_pkt_len;
+static uint32_t max_empty_polls;
+static uint32_t pause_duration;
+static uint32_t scale_freq_min;
+static uint32_t scale_freq_max;
 
 static struct rte_mempool * pktmbuf_pool[NB_SOCKETS];
 
@@ -1626,10 +1630,32 @@ print_usage(const char *prgname)
 		" empty polls, full polls, and core busyness to telemetry\n"
 		" --interrupt-only: enable interrupt-only mode\n"
 		" --pmd-mgmt MODE: enable PMD power management mode. "
-		"Currently supported modes: baseline, monitor, pause, scale\n",
+		"Currently supported modes: baseline, monitor, pause, scale\n"
+		"  --max-empty-polls MAX_EMPTY_POLLS: number of empty polls to"
+		" wait before entering sleep state\n"
+		"  --pause-duration DURATION: set the duration, in microseconds,"
+		" of the pause callback\n"
+		"  --scale-freq-min FREQ_MIN: set minimum frequency for scaling"
+		" mode for all application lcores\n"
+		"  --scale-freq-max FREQ_MAX: set maximum frequency for scaling"
+		" mode for all application lcores\n",
 		prgname);
 }
 
+static int
+parse_int(const char *opt)
+{
+	char *end = NULL;
+	unsigned long val;
+
+	/* parse integer string */
+	val = strtoul(opt, &end, 10);
+	if ((opt[0] == '\0') || (end == NULL) || (*end != '\0'))
+		return -1;
+
+	return val;
+}
+
 static int parse_max_pkt_len(const char *pktlen)
 {
 	char *end = NULL;
@@ -1803,6 +1829,10 @@ parse_ep_config(const char *q_arg)
 #define CMD_LINE_OPT_TELEMETRY "telemetry"
 #define CMD_LINE_OPT_PMD_MGMT "pmd-mgmt"
 #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len"
+#define CMD_LINE_OPT_MAX_EMPTY_POLLS "max-empty-polls"
+#define CMD_LINE_OPT_PAUSE_DURATION "pause-duration"
+#define CMD_LINE_OPT_SCALE_FREQ_MIN "scale-freq-min"
+#define CMD_LINE_OPT_SCALE_FREQ_MAX "scale-freq-max"
 
 /* Parse the argument given in the command line of the application */
 static int
@@ -1812,6 +1842,7 @@ parse_args(int argc, char **argv)
 	char **argvopt;
 	int option_index;
 	uint32_t limit;
+	int i;
 	char *prgname = argv[0];
 	static struct option lgopts[] = {
 		{"config", 1, 0, 0},
@@ -1825,6 +1856,10 @@ parse_args(int argc, char **argv)
 		{CMD_LINE_OPT_TELEMETRY, 0, 0, 0},
 		{CMD_LINE_OPT_INTERRUPT_ONLY, 0, 0, 0},
 		{CMD_LINE_OPT_PMD_MGMT, 1, 0, 0},
+		{CMD_LINE_OPT_MAX_EMPTY_POLLS, 1, 0, 0},
+		{CMD_LINE_OPT_PAUSE_DURATION, 1, 0, 0},
+		{CMD_LINE_OPT_SCALE_FREQ_MIN, 1, 0, 0},
+		{CMD_LINE_OPT_SCALE_FREQ_MAX, 1, 0, 0},
 		{NULL, 0, 0, 0}
 	};
 
@@ -1975,6 +2010,44 @@ parse_args(int argc, char **argv)
 				parse_ptype = 1;
 			}
 
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_MAX_EMPTY_POLLS,
+					sizeof(CMD_LINE_OPT_MAX_EMPTY_POLLS))) {
+				printf("Maximum empty polls configured\n");
+				max_empty_polls = parse_int(optarg);
+				rte_power_pmd_mgmt_set_emptypoll_max(max_empty_polls);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_PAUSE_DURATION,
+					sizeof(CMD_LINE_OPT_PAUSE_DURATION))) {
+				printf("Pause duration configured\n");
+				pause_duration = parse_int(optarg);
+				rte_power_pmd_mgmt_set_pause_duration(pause_duration);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_SCALE_FREQ_MIN,
+					sizeof(CMD_LINE_OPT_SCALE_FREQ_MIN))) {
+				printf("Scaling frequency minimum configured\n");
+				scale_freq_min = parse_int(optarg);
+				for (i = 0; i < RTE_MAX_LCORE; i++)
+					if (rte_lcore_is_enabled(i))
+						rte_power_pmd_mgmt_set_scaling_freq_min(i,
+								scale_freq_min);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_SCALE_FREQ_MAX,
+					sizeof(CMD_LINE_OPT_SCALE_FREQ_MAX))) {
+				printf("Scaling frequency maximum configured\n");
+				scale_freq_max = parse_int(optarg);
+				for (i = 0; i < RTE_MAX_LCORE; i++)
+					if (rte_lcore_is_enabled(i))
+						rte_power_pmd_mgmt_set_scaling_freq_max(i,
+								scale_freq_max);
+			}
+
 			break;
 
 		default:
-- 
2.31.1

[PATCH v2 0/4] Add APIs for configurable power options

[Kevin Laatz]

The power library contains some variables which are currently set by
defines, hard-coded values or set using sysfs values. In order to
configure these, code changes and recompiles are required, making
configuring these variables tedious.

This patchset introduces some new get/set APIs which allow users and
applications to configure there settings to suit their use-cases.
In addition, CLI options have been added to l3fwd_power to demonstrate
how an application could use these APIs to expose the options to users
without needing code changes to configure them.

---
v2:
* add doc update for l3fwd-power
* order version.map additions alphabetically

Kevin Laatz (4):
  lib/power: add get and set API for emptypoll max
  lib/power: add get and set API for pause duration
  lib/power: add get and set API for scaling freq min and max with
    pstate mode
  examples/l3fwd_power: add cli for configurable options

 .../sample_app_ug/l3_forward_power_man.rst    |   8 +
 examples/l3fwd-power/main.c                   |  75 +++++++++-
 lib/power/power_pstate_cpufreq.c              |  22 ++-
 lib/power/rte_power_pmd_mgmt.c                | 112 +++++++++++++-
 lib/power/rte_power_pmd_mgmt.h                | 138 ++++++++++++++++++
 lib/power/version.map                         |  10 ++
 6 files changed, 354 insertions(+), 11 deletions(-)

-- 
2.31.1

[PATCH v2 1/4] lib/power: add get and set API for emptypoll max

[Kevin Laatz]

Add new get/set APIs to configure emptypoll max which is used to
determine when a queue can go into sleep state.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
---
 lib/power/rte_power_pmd_mgmt.c | 21 ++++++++++++++++++---
 lib/power/rte_power_pmd_mgmt.h | 27 +++++++++++++++++++++++++++
 lib/power/version.map          |  4 ++++
 3 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index 39a2b4cd23..dfb7ca9187 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -11,7 +11,7 @@
 
 #include "rte_power_pmd_mgmt.h"
 
-#define EMPTYPOLL_MAX  512
+unsigned int emptypoll_max;
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -206,7 +206,7 @@ queue_can_sleep(struct pmd_core_cfg *cfg, struct queue_list_entry *qcfg)
 	qcfg->n_empty_polls++;
 
 	/* if we haven't reached threshold for empty polls, we can't sleep */
-	if (qcfg->n_empty_polls <= EMPTYPOLL_MAX)
+	if (qcfg->n_empty_polls <= emptypoll_max)
 		return false;
 
 	/*
@@ -290,7 +290,7 @@ clb_umwait(uint16_t port_id, uint16_t qidx, struct rte_mbuf **pkts __rte_unused,
 	/* this callback can't do more than one queue, omit multiqueue logic */
 	if (unlikely(nb_rx == 0)) {
 		queue_conf->n_empty_polls++;
-		if (unlikely(queue_conf->n_empty_polls > EMPTYPOLL_MAX)) {
+		if (unlikely(queue_conf->n_empty_polls > emptypoll_max)) {
 			struct rte_power_monitor_cond pmc;
 			int ret;
 
@@ -661,6 +661,18 @@ rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
 	return 0;
 }
 
+void
+rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max)
+{
+	emptypoll_max = max;
+}
+
+unsigned int
+rte_power_pmd_mgmt_get_emptypoll_max(void)
+{
+	return emptypoll_max;
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
 
@@ -669,4 +681,7 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 		struct pmd_core_cfg *cfg = &lcore_cfgs[i];
 		TAILQ_INIT(&cfg->head);
 	}
+
+	/* initialize config defaults */
+	emptypoll_max = 512;
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index 444e7b8a66..d5a94f8187 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -90,6 +90,33 @@ int
 rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
 		uint16_t port_id, uint16_t queue_id);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set a emptypoll_max to specified value. Used to specify the number of empty
+ * polls to wait before entering sleep state.
+ *
+ * @param max
+ *   The value to set emptypoll_max to.
+ */
+__rte_experimental
+void
+rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current value of emptypoll_max.
+ *
+ * @return
+ *   The current emptypoll_max value
+ */
+__rte_experimental
+unsigned int
+rte_power_pmd_mgmt_get_emptypoll_max(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 6ec6d5d96d..812843c3f3 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -38,4 +38,8 @@ EXPERIMENTAL {
 	# added in 21.02
 	rte_power_ethdev_pmgmt_queue_disable;
 	rte_power_ethdev_pmgmt_queue_enable;
+
+	# added in 22.07
+	rte_power_pmd_mgmt_get_emptypoll_max;
+	rte_power_pmd_mgmt_set_emptypoll_max;
 };
-- 
2.31.1

Re: [PATCH v2 1/4] lib/power: add get and set API for emptypoll max

[Ray Kinsella]

Kevin Laatz <kevin.laatz@intel.com> writes:

> Add new get/set APIs to configure emptypoll max which is used to
> determine when a queue can go into sleep state.
>
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> ---
>  lib/power/rte_power_pmd_mgmt.c | 21 ++++++++++++++++++---
>  lib/power/rte_power_pmd_mgmt.h | 27 +++++++++++++++++++++++++++
>  lib/power/version.map          |  4 ++++
>  3 files changed, 49 insertions(+), 3 deletions(-)
>
Acked-by: Ray Kinsella <mdr@ashroe.eu>

[PATCH v2 2/4] lib/power: add get and set API for pause duration

[Kevin Laatz]

Add new get/set API for configuring 'pause_duration' which used to adjust
the pause mode callback duration.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
---
 lib/power/rte_power_pmd_mgmt.c | 26 ++++++++++++++++++++++++--
 lib/power/rte_power_pmd_mgmt.h | 31 +++++++++++++++++++++++++++++++
 lib/power/version.map          |  2 ++
 3 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index dfb7ca9187..ab92a93aa0 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -12,6 +12,7 @@
 #include "rte_power_pmd_mgmt.h"
 
 unsigned int emptypoll_max;
+unsigned int pause_duration;
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -315,6 +316,7 @@ clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,
 	struct queue_list_entry *queue_conf = arg;
 	struct pmd_core_cfg *lcore_conf;
 	const bool empty = nb_rx == 0;
+	uint32_t pause_duration = rte_power_pmd_mgmt_get_pause_duration();
 
 	lcore_conf = &lcore_cfgs[lcore];
 
@@ -334,11 +336,11 @@ clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,
 		if (global_data.intrinsics_support.power_pause) {
 			const uint64_t cur = rte_rdtsc();
 			const uint64_t wait_tsc =
-					cur + global_data.tsc_per_us;
+					cur + global_data.tsc_per_us * pause_duration;
 			rte_power_pause(wait_tsc);
 		} else {
 			uint64_t i;
-			for (i = 0; i < global_data.pause_per_us; i++)
+			for (i = 0; i < global_data.pause_per_us * pause_duration; i++)
 				rte_pause();
 		}
 	}
@@ -673,6 +675,25 @@ rte_power_pmd_mgmt_get_emptypoll_max(void)
 	return emptypoll_max;
 }
 
+int
+rte_power_pmd_mgmt_set_pause_duration(unsigned int duration)
+{
+	if (duration == 0) {
+		printf("Pause duration must be greater than 0, value unchanged\n");
+		rte_errno = EINVAL;
+		return -1;
+	}
+	pause_duration = duration;
+
+	return 0;
+}
+
+unsigned int
+rte_power_pmd_mgmt_get_pause_duration(void)
+{
+	return pause_duration;
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
 
@@ -684,4 +705,5 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 
 	/* initialize config defaults */
 	emptypoll_max = 512;
+	pause_duration = 1;
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index d5a94f8187..18a9c3abb5 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -117,6 +117,37 @@ __rte_experimental
 unsigned int
 rte_power_pmd_mgmt_get_emptypoll_max(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the pause_duration. Used to adjust the pause mode callback duration.
+ *
+ * @note Duration must be greater than zero.
+ *
+ * @param duration
+ *   The value to set pause_duration to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_pause_duration(unsigned int duration);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current value of pause_duration.
+ *
+ * @return
+ *   The current pause_duration value.
+ */
+__rte_experimental
+unsigned int
+rte_power_pmd_mgmt_get_pause_duration(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 812843c3f3..4673b719f9 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -41,5 +41,7 @@ EXPERIMENTAL {
 
 	# added in 22.07
 	rte_power_pmd_mgmt_get_emptypoll_max;
+	rte_power_pmd_mgmt_get_pause_duration;
 	rte_power_pmd_mgmt_set_emptypoll_max;
+	rte_power_pmd_mgmt_set_pause_duration;
 };
-- 
2.31.1

Re: [PATCH v2 2/4] lib/power: add get and set API for pause duration

[Ray Kinsella]

Kevin Laatz <kevin.laatz@intel.com> writes:

> Add new get/set API for configuring 'pause_duration' which used to adjust
> the pause mode callback duration.
>
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> ---
>  lib/power/rte_power_pmd_mgmt.c | 26 ++++++++++++++++++++++++--
>  lib/power/rte_power_pmd_mgmt.h | 31 +++++++++++++++++++++++++++++++
>  lib/power/version.map          |  2 ++
>  3 files changed, 57 insertions(+), 2 deletions(-)
>
Acked-by: Ray Kinsella <mdr@ashroe.eu>

Re: [PATCH v2 2/4] lib/power: add get and set API for pause duration

[Burakov, Anatoly]

On 19-Apr-22 12:24 PM, Kevin Laatz wrote:
> Add new get/set API for configuring 'pause_duration' which used to adjust
> the pause mode callback duration.
> 
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> ---


> @@ -334,11 +336,11 @@ clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,
>   		if (global_data.intrinsics_support.power_pause) {
>   			const uint64_t cur = rte_rdtsc();
>   			const uint64_t wait_tsc =
> -					cur + global_data.tsc_per_us;
> +					cur + global_data.tsc_per_us * pause_duration;
>   			rte_power_pause(wait_tsc);
>   		} else {
>   			uint64_t i;
> -			for (i = 0; i < global_data.pause_per_us; i++)
> +			for (i = 0; i < global_data.pause_per_us * pause_duration; i++)
>   				rte_pause();
>   		}
>   	}
> @@ -673,6 +675,25 @@ rte_power_pmd_mgmt_get_emptypoll_max(void)
>   	return emptypoll_max;
>   }
>   
> +int
> +rte_power_pmd_mgmt_set_pause_duration(unsigned int duration)
> +{
> +	if (duration == 0) {
> +		printf("Pause duration must be greater than 0, value unchanged\n");

This should perhaps be an RTE_LOG rather than a printf?

With that fix,

Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

-- 
Thanks,
Anatoly

[PATCH v2 3/4] lib/power: add get and set API for scaling freq min and max with pstate mode

[Kevin Laatz]

Add new get/set API to allow the user or application to set the minimum
and maximum frequencies to use when scaling.
Previously, the frequency range was determined by the HW capabilities of
the CPU. With this new API, the user or application can constrain this
if required.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
---
 lib/power/power_pstate_cpufreq.c | 22 +++++++--
 lib/power/rte_power_pmd_mgmt.c   | 65 ++++++++++++++++++++++++++
 lib/power/rte_power_pmd_mgmt.h   | 80 ++++++++++++++++++++++++++++++++
 lib/power/version.map            |  4 ++
 4 files changed, 166 insertions(+), 5 deletions(-)

diff --git a/lib/power/power_pstate_cpufreq.c b/lib/power/power_pstate_cpufreq.c
index f4c36179ec..db10deafe2 100644
--- a/lib/power/power_pstate_cpufreq.c
+++ b/lib/power/power_pstate_cpufreq.c
@@ -12,6 +12,7 @@
 
 #include <rte_memcpy.h>
 
+#include "rte_power_pmd_mgmt.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
 
@@ -354,6 +355,7 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	FILE *f_min = NULL, *f_max = NULL;
 	int ret = -1;
 	uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
+	int config_min_freq, config_max_freq;
 	uint32_t i, num_freqs = 0;
 
 	/* open all files */
@@ -388,6 +390,16 @@ power_get_available_freqs(struct pstate_power_info *pi)
 		goto out;
 	}
 
+	/* check for config set by user or application to limit frequency range */
+	config_min_freq = rte_power_pmd_mgmt_get_scaling_freq_min(pi->lcore_id);
+	if (config_min_freq < 0)
+		goto out;
+	config_max_freq = rte_power_pmd_mgmt_get_scaling_freq_max(pi->lcore_id);
+	if (config_max_freq < 0)
+		goto out;
+	sys_min_freq = RTE_MAX(sys_min_freq, (uint32_t)config_min_freq);
+	sys_max_freq = RTE_MIN(sys_max_freq, (uint32_t)config_max_freq);
+
 	if (sys_max_freq < sys_min_freq)
 		goto out;
 
@@ -411,8 +423,8 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	/* If turbo is available then there is one extra freq bucket
 	 * to store the sys max freq which value is base_max +1
 	 */
-	num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
-		pi->turbo_available;
+	num_freqs = (RTE_MIN(base_max_freq, sys_max_freq) - sys_min_freq) / BUS_FREQ
+			+ 1 + pi->turbo_available;
 	if (num_freqs >= RTE_MAX_LCORE_FREQS) {
 		RTE_LOG(ERR, POWER, "Too many available frequencies: %d\n",
 				num_freqs);
@@ -427,10 +439,10 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	 */
 	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 		if ((i == 0) && pi->turbo_available)
-			pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
+			pi->freqs[pi->nb_freqs++] = RTE_MIN(base_max_freq, sys_max_freq) + 1;
 		else
-			pi->freqs[pi->nb_freqs++] =
-			base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
+			pi->freqs[pi->nb_freqs++] = RTE_MIN(base_max_freq, sys_max_freq) -
+					(i - pi->turbo_available) * BUS_FREQ;
 	}
 
 	ret = 0;
diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index ab92a93aa0..5ba050dc95 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -10,9 +10,12 @@
 #include <rte_power_intrinsics.h>
 
 #include "rte_power_pmd_mgmt.h"
+#include "power_common.h"
 
 unsigned int emptypoll_max;
 unsigned int pause_duration;
+unsigned int scale_freq_min[RTE_MAX_LCORE];
+unsigned int scale_freq_max[RTE_MAX_LCORE];
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -694,8 +697,65 @@ rte_power_pmd_mgmt_get_pause_duration(void)
 	return pause_duration;
 }
 
+int
+rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		rte_errno = EINVAL;
+		return -1;
+	}
+	scale_freq_min[lcore] = min;
+
+	return 0;
+}
+
+int
+rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		rte_errno = EINVAL;
+		return -1;
+	}
+	scale_freq_max[lcore] = max;
+
+	return 0;
+}
+
+int
+rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		rte_errno = EINVAL;
+		return -1;
+	}
+
+	if (scale_freq_max[lcore] == 0)
+		RTE_LOG(DEBUG, POWER, "Scaling freq min config not set. Using sysfs min freq.\n");
+
+	return scale_freq_min[lcore];
+}
+
+int
+rte_power_pmd_mgmt_get_scaling_freq_max(unsigned int lcore)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		rte_errno = EINVAL;
+		return -1;
+	}
+
+	if (scale_freq_max[lcore] == UINT32_MAX)
+		RTE_LOG(DEBUG, POWER, "Scaling freq max config not set. Using sysfs max freq.\n");
+
+	return scale_freq_max[lcore];
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
+	int j;
 
 	/* initialize all tailqs */
 	for (i = 0; i < RTE_DIM(lcore_cfgs); i++) {
@@ -706,4 +766,9 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	/* initialize config defaults */
 	emptypoll_max = 512;
 	pause_duration = 1;
+	/* scaling defaults out of range to ensure not used unless set by user or app */
+	for (j = 0; j < RTE_MAX_LCORE; j++) {
+		scale_freq_min[j] = 0;
+		scale_freq_max[j] = UINT32_MAX;
+	}
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index 18a9c3abb5..74e3fa710b 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -148,6 +148,86 @@ __rte_experimental
 unsigned int
 rte_power_pmd_mgmt_get_pause_duration(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the min frequency to be used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to set the min frequency for.
+ * @param min
+ *   The value, in Hertz, to set the minimum frequency to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the max frequency to be used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to set the max frequency for.
+ * @param max
+ *   The value, in Hertz, to set the maximum frequency to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current configured min frequency used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to get the min frequency for.
+ * @return
+ *   0 if no value has been configured via the 'set' API.
+ *   >0 if a minimum frequency has been configured. Value is the minimum frequency
+ *   , in Hertz, used for frequency scaling.
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current configured max frequency used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to get the max frequency for.
+ * @return
+ *   UINT32_MAX if no value has been configured via the 'set' API.
+ *   On success, the current configured maximum frequency, in Hertz, used for
+ *   frequency scaling..
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_get_scaling_freq_max(unsigned int lcore);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 4673b719f9..a687754f4a 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -42,6 +42,10 @@ EXPERIMENTAL {
 	# added in 22.07
 	rte_power_pmd_mgmt_get_emptypoll_max;
 	rte_power_pmd_mgmt_get_pause_duration;
+	rte_power_pmd_mgmt_get_scaling_freq_max;
+	rte_power_pmd_mgmt_get_scaling_freq_min;
 	rte_power_pmd_mgmt_set_emptypoll_max;
 	rte_power_pmd_mgmt_set_pause_duration;
+	rte_power_pmd_mgmt_set_scaling_freq_max;
+	rte_power_pmd_mgmt_set_scaling_freq_min;
 };
-- 
2.31.1

Re: [PATCH v2 3/4] lib/power: add get and set API for scaling freq min and max with pstate mode

[Ray Kinsella]

Kevin Laatz <kevin.laatz@intel.com> writes:

> Add new get/set API to allow the user or application to set the minimum
> and maximum frequencies to use when scaling.
> Previously, the frequency range was determined by the HW capabilities of
> the CPU. With this new API, the user or application can constrain this
> if required.
>
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> ---
>  lib/power/power_pstate_cpufreq.c | 22 +++++++--
>  lib/power/rte_power_pmd_mgmt.c   | 65 ++++++++++++++++++++++++++
>  lib/power/rte_power_pmd_mgmt.h   | 80 ++++++++++++++++++++++++++++++++
>  lib/power/version.map            |  4 ++
>  4 files changed, 166 insertions(+), 5 deletions(-)
>

Acked-by: Ray Kinsella <mdr@ashroe.eu>

Re: [PATCH v2 3/4] lib/power: add get and set API for scaling freq min and max with pstate mode

[Burakov, Anatoly]

On 19-Apr-22 12:25 PM, Kevin Laatz wrote:
> Add new get/set API to allow the user or application to set the minimum
> and maximum frequencies to use when scaling.
> Previously, the frequency range was determined by the HW capabilities of
> the CPU. With this new API, the user or application can constrain this
> if required.
> 
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> ---

<snip>

>   
> +int
> +rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min)
> +{
> +	if (lcore >= RTE_MAX_LCORE) {
> +		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	scale_freq_min[lcore] = min;

Are there any constraints on the value ranges, or are we just going to 
accept any and all values? If the idea was to allow valid values plus 
some special "default" value, you can still restrict the range, but 
allow 0 as a special case?

> +
> +	return 0;
> +}
> +
> +int
> +rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max)
> +{
> +	if (lcore >= RTE_MAX_LCORE) {
> +		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	scale_freq_max[lcore] = max;

Same as above. Also, do we want UINT32_MAX be the "special" value for 
the "max" case? What do you think of having "0" as "not set", but maybe 
set it internally to UINT32_MAX if you still want to keep using the 
RTE_MIN/MAX macros?

> +
> +	return 0;
> +}
> +
> +int
> +rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore)
> +{

<snip>

> diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
> index 18a9c3abb5..74e3fa710b 100644
> --- a/lib/power/rte_power_pmd_mgmt.h
> +++ b/lib/power/rte_power_pmd_mgmt.h
> @@ -148,6 +148,86 @@ __rte_experimental
>   unsigned int
>   rte_power_pmd_mgmt_get_pause_duration(void);
>   
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
> + *
> + * Set the min frequency to be used for frequency scaling.
> + *
> + * @note Supported by: Pstate mode.
> + *
> + * @param lcore
> + *   The ID of the lcore to set the min frequency for.
> + * @param min
> + *   The value, in Hertz, to set the minimum frequency to.

Is it really in Hertz? As far as I can tell, it's in steps of 100MHz 
(BUS_FREQ).

-- 
Thanks,
Anatoly

Re: [PATCH v2 3/4] lib/power: add get and set API for scaling freq min and max with pstate mode

[Kevin Laatz]

On 18/05/2022 10:05, Burakov, Anatoly wrote:
> On 19-Apr-22 12:25 PM, Kevin Laatz wrote:
>> Add new get/set API to allow the user or application to set the minimum
>> and maximum frequencies to use when scaling.
>> Previously, the frequency range was determined by the HW capabilities of
>> the CPU. With this new API, the user or application can constrain this
>> if required.
>>
>> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
>> ---
>
> <snip>
>
>>   +int
>> +rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned 
>> int min)
>> +{
>> +    if (lcore >= RTE_MAX_LCORE) {
>> +        RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
>> +        rte_errno = EINVAL;
>> +        return -1;
>> +    }
>> +    scale_freq_min[lcore] = min;
>
> Are there any constraints on the value ranges, or are we just going to 
> accept any and all values? If the idea was to allow valid values plus 
> some special "default" value, you can still restrict the range, but 
> allow 0 as a special case?

When writing min/max values to HW the values are clamped. Since the API 
takes unsigned integer for the frequency value (in this case 'min'), any 
value can be considered as 'valid'.

That being said, this should at least check that min <= max for the same 
lcore. I'll add this for v3.


>
>> +
>> +    return 0;
>> +}
>> +
>> +int
>> +rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned 
>> int max)
>> +{
>> +    if (lcore >= RTE_MAX_LCORE) {
>> +        RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
>> +        rte_errno = EINVAL;
>> +        return -1;
>> +    }
>> +    scale_freq_max[lcore] = max;
>
> Same as above. Also, do we want UINT32_MAX be the "special" value for 
> the "max" case? What do you think of having "0" as "not set", but 
> maybe set it internally to UINT32_MAX if you still want to keep using 
> the RTE_MIN/MAX macros?

Similar to  'set_scaling_freq_min', the value will be clamped by HW so 
any value can be considered 'valid'. I don't see the benefit of having 
"0" for not set, since UINT32_MAX will achieve the same result, i.e. the 
value won't be used (it will fall back the max value in sysfs). Do you 
have a use-case for it if we don't need a 'special case'?

Will add a check to make sure max >= min for v3.


>
>> +
>> +    return 0;
>> +}
>> +
>> +int
>> +rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore)
>> +{
>
> <snip>
>
>> diff --git a/lib/power/rte_power_pmd_mgmt.h 
>> b/lib/power/rte_power_pmd_mgmt.h
>> index 18a9c3abb5..74e3fa710b 100644
>> --- a/lib/power/rte_power_pmd_mgmt.h
>> +++ b/lib/power/rte_power_pmd_mgmt.h
>> @@ -148,6 +148,86 @@ __rte_experimental
>>   unsigned int
>>   rte_power_pmd_mgmt_get_pause_duration(void);
>>   +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change, or be removed, without 
>> prior notice.
>> + *
>> + * Set the min frequency to be used for frequency scaling.
>> + *
>> + * @note Supported by: Pstate mode.
>> + *
>> + * @param lcore
>> + *   The ID of the lcore to set the min frequency for.
>> + * @param min
>> + *   The value, in Hertz, to set the minimum frequency to.
>
> Is it really in Hertz? As far as I can tell, it's in steps of 100MHz 
> (BUS_FREQ).

Correct, the frequency changes in steps of 100MHz, but the value passed 
to 'min' is in kHz - will ammend the comments.

[PATCH v2 4/4] examples/l3fwd_power: add cli for configurable options

[Kevin Laatz]

Add CLI options to l3fwd_power to utilize the new power APIs introduced in
this patchset. These CLI options allow the user to configure the
heuritstics made available through the new API via the l3fwd_power
application options.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>

---
v2: add doc update for l3fwd-power
---
 .../sample_app_ug/l3_forward_power_man.rst    |  8 ++
 examples/l3fwd-power/main.c                   | 75 ++++++++++++++++++-
 2 files changed, 82 insertions(+), 1 deletion(-)

diff --git a/doc/guides/sample_app_ug/l3_forward_power_man.rst b/doc/guides/sample_app_ug/l3_forward_power_man.rst
index 2e350c45f1..b26f11a220 100644
--- a/doc/guides/sample_app_ug/l3_forward_power_man.rst
+++ b/doc/guides/sample_app_ug/l3_forward_power_man.rst
@@ -109,6 +109,14 @@ where,
 
 *   --pmd-mgmt: PMD power management mode.
 
+*   --max-empty-polls : Number of empty polls to wait before entering sleep state.
+
+*   --pause-duration: Set the duration of the pause callback (microseconds).
+
+*   --scale-freq-min: Set minimum frequency for scaling.
+
+*   --scale-freq-max: Set maximum frequency for scaling.
+
 See :doc:`l3_forward` for details.
 The L3fwd-power example reuses the L3fwd command line options.
 
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 20e5b59af9..ec2a71f02f 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -265,6 +265,10 @@ static struct rte_eth_conf port_conf = {
 };
 
 static uint32_t max_pkt_len;
+static uint32_t max_empty_polls;
+static uint32_t pause_duration;
+static uint32_t scale_freq_min;
+static uint32_t scale_freq_max;
 
 static struct rte_mempool * pktmbuf_pool[NB_SOCKETS];
 
@@ -1626,10 +1630,32 @@ print_usage(const char *prgname)
 		" empty polls, full polls, and core busyness to telemetry\n"
 		" --interrupt-only: enable interrupt-only mode\n"
 		" --pmd-mgmt MODE: enable PMD power management mode. "
-		"Currently supported modes: baseline, monitor, pause, scale\n",
+		"Currently supported modes: baseline, monitor, pause, scale\n"
+		"  --max-empty-polls MAX_EMPTY_POLLS: number of empty polls to"
+		" wait before entering sleep state\n"
+		"  --pause_duration DURATION: set the duration, in microseconds,"
+		" of the pause callback\n"
+		"  --scale-freq-min FREQ_MIN: set minimum frequency for scaling"
+		" mode for all application lcores\n"
+		"  --scale_freq_max FREQ_MAX: set maximum frequency for scaling"
+		" mode for all application lcores\n",
 		prgname);
 }
 
+static int
+parse_int(const char *opt)
+{
+	char *end = NULL;
+	unsigned long val;
+
+	/* parse integer string */
+	val = strtoul(opt, &end, 10);
+	if ((opt[0] == '\0') || (end == NULL) || (*end != '\0'))
+		return -1;
+
+	return val;
+}
+
 static int parse_max_pkt_len(const char *pktlen)
 {
 	char *end = NULL;
@@ -1803,6 +1829,10 @@ parse_ep_config(const char *q_arg)
 #define CMD_LINE_OPT_TELEMETRY "telemetry"
 #define CMD_LINE_OPT_PMD_MGMT "pmd-mgmt"
 #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len"
+#define CMD_LINE_OPT_MAX_EMPTY_POLLS "max-empty-poll"
+#define CMD_LINE_OPT_PAUSE_DURATION "pause-duration"
+#define CMD_LINE_OPT_SCALE_FREQ_MIN "scale-freq-min"
+#define CMD_LINE_OPT_SCALE_FREQ_MAX "scale-freq-max"
 
 /* Parse the argument given in the command line of the application */
 static int
@@ -1812,6 +1842,7 @@ parse_args(int argc, char **argv)
 	char **argvopt;
 	int option_index;
 	uint32_t limit;
+	int i;
 	char *prgname = argv[0];
 	static struct option lgopts[] = {
 		{"config", 1, 0, 0},
@@ -1825,6 +1856,10 @@ parse_args(int argc, char **argv)
 		{CMD_LINE_OPT_TELEMETRY, 0, 0, 0},
 		{CMD_LINE_OPT_INTERRUPT_ONLY, 0, 0, 0},
 		{CMD_LINE_OPT_PMD_MGMT, 1, 0, 0},
+		{CMD_LINE_OPT_MAX_EMPTY_POLLS, 1, 0, 0},
+		{CMD_LINE_OPT_PAUSE_DURATION, 1, 0, 0},
+		{CMD_LINE_OPT_SCALE_FREQ_MIN, 1, 0, 0},
+		{CMD_LINE_OPT_SCALE_FREQ_MAX, 1, 0, 0},
 		{NULL, 0, 0, 0}
 	};
 
@@ -1975,6 +2010,44 @@ parse_args(int argc, char **argv)
 				parse_ptype = 1;
 			}
 
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_MAX_EMPTY_POLLS,
+					sizeof(CMD_LINE_OPT_MAX_EMPTY_POLLS))) {
+				printf("Maximum empty polls configured\n");
+				max_empty_polls = parse_int(optarg);
+				rte_power_pmd_mgmt_set_emptypoll_max(max_empty_polls);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_PAUSE_DURATION,
+					sizeof(CMD_LINE_OPT_PAUSE_DURATION))) {
+				printf("Pause duration configured\n");
+				pause_duration = parse_int(optarg);
+				rte_power_pmd_mgmt_set_pause_duration(pause_duration);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_SCALE_FREQ_MIN,
+					sizeof(CMD_LINE_OPT_SCALE_FREQ_MIN))) {
+				printf("Scaling frequency minimum configured\n");
+				scale_freq_min = parse_int(optarg);
+				for (i = 0; i < RTE_MAX_LCORE; i++)
+					if (rte_lcore_is_enabled(i))
+						rte_power_pmd_mgmt_set_scaling_freq_min(i,
+								scale_freq_min);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_SCALE_FREQ_MAX,
+					sizeof(CMD_LINE_OPT_SCALE_FREQ_MAX))) {
+				printf("Scaling frequency maximum configured\n");
+				scale_freq_max = parse_int(optarg);
+				for (i = 0; i < RTE_MAX_LCORE; i++)
+					if (rte_lcore_is_enabled(i))
+						rte_power_pmd_mgmt_set_scaling_freq_max(i,
+								scale_freq_max);
+			}
+
 			break;
 
 		default:
-- 
2.31.1

Re: [PATCH v2 4/4] examples/l3fwd_power: add cli for configurable options

[Burakov, Anatoly]

On 19-Apr-22 12:25 PM, Kevin Laatz wrote:
> Add CLI options to l3fwd_power to utilize the new power APIs introduced in
> this patchset. These CLI options allow the user to configure the
> heuritstics made available through the new API via the l3fwd_power
> application options.
> 
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> 
> ---
> v2: add doc update for l3fwd-power
> ---

This is a bit confusing. There are other scaling modes in l3fwd-power, 
but the --scale-freq-min only applies to the PMD power management mode.

>   .../sample_app_ug/l3_forward_power_man.rst    |  8 ++
>   examples/l3fwd-power/main.c                   | 75 ++++++++++++++++++-
>   2 files changed, 82 insertions(+), 1 deletion(-)
> 
> diff --git a/doc/guides/sample_app_ug/l3_forward_power_man.rst b/doc/guides/sample_app_ug/l3_forward_power_man.rst
> index 2e350c45f1..b26f11a220 100644
> --- a/doc/guides/sample_app_ug/l3_forward_power_man.rst
> +++ b/doc/guides/sample_app_ug/l3_forward_power_man.rst
> @@ -109,6 +109,14 @@ where,
>   
>   *   --pmd-mgmt: PMD power management mode.
>   
> +*   --max-empty-polls : Number of empty polls to wait before entering sleep state.
> +
> +*   --pause-duration: Set the duration of the pause callback (microseconds).
> +
> +*   --scale-freq-min: Set minimum frequency for scaling.
> +
> +*   --scale-freq-max: Set maximum frequency for scaling.
> +

Maybe make a note that these only apply to --pmd-mgmt mode?

>   See :doc:`l3_forward` for details.
>   The L3fwd-power example reuses the L3fwd command line options.
>   
> diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
> index 20e5b59af9..ec2a71f02f 100644
> --- a/examples/l3fwd-power/main.c
> +++ b/examples/l3fwd-power/main.c
> @@ -265,6 +265,10 @@ static struct rte_eth_conf port_conf = {
>   };
>   
>   static uint32_t max_pkt_len;
> +static uint32_t max_empty_polls;
> +static uint32_t pause_duration;
> +static uint32_t scale_freq_min;
> +static uint32_t scale_freq_max;
>   
>   static struct rte_mempool * pktmbuf_pool[NB_SOCKETS];
>   
> @@ -1626,10 +1630,32 @@ print_usage(const char *prgname)
>   		" empty polls, full polls, and core busyness to telemetry\n"
>   		" --interrupt-only: enable interrupt-only mode\n"
>   		" --pmd-mgmt MODE: enable PMD power management mode. "
> -		"Currently supported modes: baseline, monitor, pause, scale\n",
> +		"Currently supported modes: baseline, monitor, pause, scale\n"
> +		"  --max-empty-polls MAX_EMPTY_POLLS: number of empty polls to"
> +		" wait before entering sleep state\n"
> +		"  --pause_duration DURATION: set the duration, in microseconds,"
> +		" of the pause callback\n"
> +		"  --scale-freq-min FREQ_MIN: set minimum frequency for scaling"
> +		" mode for all application lcores\n"
> +		"  --scale_freq_max FREQ_MAX: set maximum frequency for scaling"
> +		" mode for all application lcores\n",

It would be useful to note which units are used here :) This is 
increments in 100MHz, correct?

>   		prgname);
>   }
>   
> +static int
> +parse_int(const char *opt)
> +{
> +	char *end = NULL;
> +	unsigned long val;
> +
> +	/* parse integer string */
> +	val = strtoul(opt, &end, 10);
> +	if ((opt[0] == '\0') || (end == NULL) || (*end != '\0'))
> +		return -1;
> +
> +	return val;
> +}
> +
>   static int parse_max_pkt_len(const char *pktlen)
>   {
>   	char *end = NULL;
> @@ -1803,6 +1829,10 @@ parse_ep_config(const char *q_arg)
>   #define CMD_LINE_OPT_TELEMETRY "telemetry"
>   #define CMD_LINE_OPT_PMD_MGMT "pmd-mgmt"
>   #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len"
> +#define CMD_LINE_OPT_MAX_EMPTY_POLLS "max-empty-poll"
> +#define CMD_LINE_OPT_PAUSE_DURATION "pause-duration"
> +#define CMD_LINE_OPT_SCALE_FREQ_MIN "scale-freq-min"
> +#define CMD_LINE_OPT_SCALE_FREQ_MAX "scale-freq-max"
>   
>   /* Parse the argument given in the command line of the application */
>   static int
> @@ -1812,6 +1842,7 @@ parse_args(int argc, char **argv)
>   	char **argvopt;
>   	int option_index;
>   	uint32_t limit;
> +	int i;
>   	char *prgname = argv[0];
>   	static struct option lgopts[] = {
>   		{"config", 1, 0, 0},
> @@ -1825,6 +1856,10 @@ parse_args(int argc, char **argv)
>   		{CMD_LINE_OPT_TELEMETRY, 0, 0, 0},
>   		{CMD_LINE_OPT_INTERRUPT_ONLY, 0, 0, 0},
>   		{CMD_LINE_OPT_PMD_MGMT, 1, 0, 0},
> +		{CMD_LINE_OPT_MAX_EMPTY_POLLS, 1, 0, 0},
> +		{CMD_LINE_OPT_PAUSE_DURATION, 1, 0, 0},
> +		{CMD_LINE_OPT_SCALE_FREQ_MIN, 1, 0, 0},
> +		{CMD_LINE_OPT_SCALE_FREQ_MAX, 1, 0, 0},
>   		{NULL, 0, 0, 0}
>   	};
>   
> @@ -1975,6 +2010,44 @@ parse_args(int argc, char **argv)
>   				parse_ptype = 1;
>   			}
>   
> +			if (!strncmp(lgopts[option_index].name,
> +					CMD_LINE_OPT_MAX_EMPTY_POLLS,
> +					sizeof(CMD_LINE_OPT_MAX_EMPTY_POLLS))) {
> +				printf("Maximum empty polls configured\n");
> +				max_empty_polls = parse_int(optarg);
> +				rte_power_pmd_mgmt_set_emptypoll_max(max_empty_polls);

If you have global variables anyway, why are you setting this here? :) 
You could call these API's under PMD mgmt mode init only, and this would 
also give you a chance of checking return values of any of these API's - 
for example, AFAIR set_pause_duration() can return an error if the value 
supplied is invalid (0).

Plus, I still feel uneasy about having DPDK API calls right inside arg 
parsing code.

> +			}
> +
> +			if (!strncmp(lgopts[option_index].name,
> +					CMD_LINE_OPT_PAUSE_DURATION,
> +					sizeof(CMD_LINE_OPT_PAUSE_DURATION))) {
> +				printf("Pause duration configured\n");
> +				pause_duration = parse_int(optarg);
> +				rte_power_pmd_mgmt_set_pause_duration(pause_duration);
> +			}
> +
> +			if (!strncmp(lgopts[option_index].name,
> +					CMD_LINE_OPT_SCALE_FREQ_MIN,
> +					sizeof(CMD_LINE_OPT_SCALE_FREQ_MIN))) {
> +				printf("Scaling frequency minimum configured\n");
> +				scale_freq_min = parse_int(optarg);
> +				for (i = 0; i < RTE_MAX_LCORE; i++)
> +					if (rte_lcore_is_enabled(i))
> +						rte_power_pmd_mgmt_set_scaling_freq_min(i,
> +								scale_freq_min);
> +			}
> +
> +			if (!strncmp(lgopts[option_index].name,
> +					CMD_LINE_OPT_SCALE_FREQ_MAX,
> +					sizeof(CMD_LINE_OPT_SCALE_FREQ_MAX))) {
> +				printf("Scaling frequency maximum configured\n");
> +				scale_freq_max = parse_int(optarg);
> +				for (i = 0; i < RTE_MAX_LCORE; i++)
> +					if (rte_lcore_is_enabled(i))
> +						rte_power_pmd_mgmt_set_scaling_freq_max(i,
> +								scale_freq_max);
> +			}
> +
>   			break;
>   
>   		default:


-- 
Thanks,
Anatoly

Re: [PATCH v2 4/4] examples/l3fwd_power: add cli for configurable options

[Kevin Laatz]

On 18/05/2022 10:11, Burakov, Anatoly wrote:
> On 19-Apr-22 12:25 PM, Kevin Laatz wrote:
>> Add CLI options to l3fwd_power to utilize the new power APIs 
>> introduced in
>> this patchset. These CLI options allow the user to configure the
>> heuritstics made available through the new API via the l3fwd_power
>> application options.
>>
>> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
>>
>> ---
>> v2: add doc update for l3fwd-power
>> ---
>
> This is a bit confusing. There are other scaling modes in l3fwd-power, 
> but the --scale-freq-min only applies to the PMD power management mode.
>
>> .../sample_app_ug/l3_forward_power_man.rst    |  8 ++
>>   examples/l3fwd-power/main.c                   | 75 ++++++++++++++++++-
>>   2 files changed, 82 insertions(+), 1 deletion(-)
>>
>> diff --git a/doc/guides/sample_app_ug/l3_forward_power_man.rst 
>> b/doc/guides/sample_app_ug/l3_forward_power_man.rst
>> index 2e350c45f1..b26f11a220 100644
>> --- a/doc/guides/sample_app_ug/l3_forward_power_man.rst
>> +++ b/doc/guides/sample_app_ug/l3_forward_power_man.rst
>> @@ -109,6 +109,14 @@ where,
>>     *   --pmd-mgmt: PMD power management mode.
>>   +*   --max-empty-polls : Number of empty polls to wait before 
>> entering sleep state.
>> +
>> +*   --pause-duration: Set the duration of the pause callback 
>> (microseconds).
>> +
>> +*   --scale-freq-min: Set minimum frequency for scaling.
>> +
>> +*   --scale-freq-max: Set maximum frequency for scaling.
>> +
>
> Maybe make a note that these only apply to --pmd-mgmt mode?

Fair point, I'll add a note.


>
>>   See :doc:`l3_forward` for details.
>>   The L3fwd-power example reuses the L3fwd command line options.
>>   diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
>> index 20e5b59af9..ec2a71f02f 100644
>> --- a/examples/l3fwd-power/main.c
>> +++ b/examples/l3fwd-power/main.c
>> @@ -265,6 +265,10 @@ static struct rte_eth_conf port_conf = {
>>   };
>>     static uint32_t max_pkt_len;
>> +static uint32_t max_empty_polls;
>> +static uint32_t pause_duration;
>> +static uint32_t scale_freq_min;
>> +static uint32_t scale_freq_max;
>>     static struct rte_mempool * pktmbuf_pool[NB_SOCKETS];
>>   @@ -1626,10 +1630,32 @@ print_usage(const char *prgname)
>>           " empty polls, full polls, and core busyness to telemetry\n"
>>           " --interrupt-only: enable interrupt-only mode\n"
>>           " --pmd-mgmt MODE: enable PMD power management mode. "
>> -        "Currently supported modes: baseline, monitor, pause, scale\n",
>> +        "Currently supported modes: baseline, monitor, pause, scale\n"
>> +        "  --max-empty-polls MAX_EMPTY_POLLS: number of empty polls to"
>> +        " wait before entering sleep state\n"
>> +        "  --pause_duration DURATION: set the duration, in 
>> microseconds,"
>> +        " of the pause callback\n"
>> +        "  --scale-freq-min FREQ_MIN: set minimum frequency for 
>> scaling"
>> +        " mode for all application lcores\n"
>> +        "  --scale_freq_max FREQ_MAX: set maximum frequency for 
>> scaling"
>> +        " mode for all application lcores\n",
>
> It would be useful to note which units are used here :) This is 
> increments in 100MHz, correct?

Ack


>
>>           prgname);
>>   }
>>   +static int
>> +parse_int(const char *opt)
>> +{
>> +    char *end = NULL;
>> +    unsigned long val;
>> +
>> +    /* parse integer string */
>> +    val = strtoul(opt, &end, 10);
>> +    if ((opt[0] == '\0') || (end == NULL) || (*end != '\0'))
>> +        return -1;
>> +
>> +    return val;
>> +}
>> +
>>   static int parse_max_pkt_len(const char *pktlen)
>>   {
>>       char *end = NULL;
>> @@ -1803,6 +1829,10 @@ parse_ep_config(const char *q_arg)
>>   #define CMD_LINE_OPT_TELEMETRY "telemetry"
>>   #define CMD_LINE_OPT_PMD_MGMT "pmd-mgmt"
>>   #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len"
>> +#define CMD_LINE_OPT_MAX_EMPTY_POLLS "max-empty-poll"
>> +#define CMD_LINE_OPT_PAUSE_DURATION "pause-duration"
>> +#define CMD_LINE_OPT_SCALE_FREQ_MIN "scale-freq-min"
>> +#define CMD_LINE_OPT_SCALE_FREQ_MAX "scale-freq-max"
>>     /* Parse the argument given in the command line of the 
>> application */
>>   static int
>> @@ -1812,6 +1842,7 @@ parse_args(int argc, char **argv)
>>       char **argvopt;
>>       int option_index;
>>       uint32_t limit;
>> +    int i;
>>       char *prgname = argv[0];
>>       static struct option lgopts[] = {
>>           {"config", 1, 0, 0},
>> @@ -1825,6 +1856,10 @@ parse_args(int argc, char **argv)
>>           {CMD_LINE_OPT_TELEMETRY, 0, 0, 0},
>>           {CMD_LINE_OPT_INTERRUPT_ONLY, 0, 0, 0},
>>           {CMD_LINE_OPT_PMD_MGMT, 1, 0, 0},
>> +        {CMD_LINE_OPT_MAX_EMPTY_POLLS, 1, 0, 0},
>> +        {CMD_LINE_OPT_PAUSE_DURATION, 1, 0, 0},
>> +        {CMD_LINE_OPT_SCALE_FREQ_MIN, 1, 0, 0},
>> +        {CMD_LINE_OPT_SCALE_FREQ_MAX, 1, 0, 0},
>>           {NULL, 0, 0, 0}
>>       };
>>   @@ -1975,6 +2010,44 @@ parse_args(int argc, char **argv)
>>                   parse_ptype = 1;
>>               }
>>   +            if (!strncmp(lgopts[option_index].name,
>> +                    CMD_LINE_OPT_MAX_EMPTY_POLLS,
>> +                    sizeof(CMD_LINE_OPT_MAX_EMPTY_POLLS))) {
>> +                printf("Maximum empty polls configured\n");
>> +                max_empty_polls = parse_int(optarg);
>> + rte_power_pmd_mgmt_set_emptypoll_max(max_empty_polls);
>
> If you have global variables anyway, why are you setting this here? :) 
> You could call these API's under PMD mgmt mode init only, and this 
> would also give you a chance of checking return values of any of these 
> API's - for example, AFAIR set_pause_duration() can return an error if 
> the value supplied is invalid (0).
>
> Plus, I still feel uneasy about having DPDK API calls right inside arg 
> parsing code.

I'll look into this and move the API calls out of the parsing for v3.

Thanks for reviewing!


>
>> +            }
>> +
>> +            if (!strncmp(lgopts[option_index].name,
>> +                    CMD_LINE_OPT_PAUSE_DURATION,
>> +                    sizeof(CMD_LINE_OPT_PAUSE_DURATION))) {
>> +                printf("Pause duration configured\n");
>> +                pause_duration = parse_int(optarg);
>> + rte_power_pmd_mgmt_set_pause_duration(pause_duration);
>> +            }
>> +
>> +            if (!strncmp(lgopts[option_index].name,
>> +                    CMD_LINE_OPT_SCALE_FREQ_MIN,
>> +                    sizeof(CMD_LINE_OPT_SCALE_FREQ_MIN))) {
>> +                printf("Scaling frequency minimum configured\n");
>> +                scale_freq_min = parse_int(optarg);
>> +                for (i = 0; i < RTE_MAX_LCORE; i++)
>> +                    if (rte_lcore_is_enabled(i))
>> + rte_power_pmd_mgmt_set_scaling_freq_min(i,
>> +                                scale_freq_min);
>> +            }
>> +
>> +            if (!strncmp(lgopts[option_index].name,
>> +                    CMD_LINE_OPT_SCALE_FREQ_MAX,
>> +                    sizeof(CMD_LINE_OPT_SCALE_FREQ_MAX))) {
>> +                printf("Scaling frequency maximum configured\n");
>> +                scale_freq_max = parse_int(optarg);
>> +                for (i = 0; i < RTE_MAX_LCORE; i++)
>> +                    if (rte_lcore_is_enabled(i))
>> + rte_power_pmd_mgmt_set_scaling_freq_max(i,
>> +                                scale_freq_max);
>> +            }
>> +
>>               break;
>>             default:
>
>

[PATCH v3 0/4] Add APIs for configurable power options

[Kevin Laatz]

The power library contains some variables which are currently set by
defines, hard-coded values or set using sysfs values. In order to
configure these, code changes and recompiles are required, making
configuring these variables tedious.

This patchset introduces some new get/set APIs which allow users and
applications to configure there settings to suit their use-cases.
In addition, CLI options have been added to l3fwd_power to demonstrate
how an application could use these APIs to expose the options to users
without needing code changes to configure them.

---
v2:
* add doc update for l3fwd-power
* order version.map additions alphabetically

v3:
* move setters from arg parse function to init
* consider 0 as 'not set' for scaling_freq_max
* other minor fixes

Kevin Laatz (4):
  lib/power: add get and set API for emptypoll max
  lib/power: add get and set API for pause duration
  lib/power: add get and set API for scaling freq min and max with
    pstate mode
  examples/l3fwd_power: add cli for configurable options

 .../sample_app_ug/l3_forward_power_man.rst    |   8 +
 examples/l3fwd-power/main.c                   |  86 ++++++++++-
 lib/power/power_pstate_cpufreq.c              |  22 ++-
 lib/power/rte_power_pmd_mgmt.c                | 121 ++++++++++++++-
 lib/power/rte_power_pmd_mgmt.h                | 139 ++++++++++++++++++
 lib/power/version.map                         |  10 ++
 6 files changed, 375 insertions(+), 11 deletions(-)

-- 
2.31.1

[PATCH v3 1/4] lib/power: add get and set API for emptypoll max

[Kevin Laatz]

Add new get/set APIs to configure emptypoll max which is used to
determine when a queue can go into sleep state.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Ray Kinsella <mdr@ashroe.eu>
---
 lib/power/rte_power_pmd_mgmt.c | 21 ++++++++++++++++++---
 lib/power/rte_power_pmd_mgmt.h | 27 +++++++++++++++++++++++++++
 lib/power/version.map          |  4 ++++
 3 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index 39a2b4cd23..dfb7ca9187 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -11,7 +11,7 @@
 
 #include "rte_power_pmd_mgmt.h"
 
-#define EMPTYPOLL_MAX  512
+unsigned int emptypoll_max;
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -206,7 +206,7 @@ queue_can_sleep(struct pmd_core_cfg *cfg, struct queue_list_entry *qcfg)
 	qcfg->n_empty_polls++;
 
 	/* if we haven't reached threshold for empty polls, we can't sleep */
-	if (qcfg->n_empty_polls <= EMPTYPOLL_MAX)
+	if (qcfg->n_empty_polls <= emptypoll_max)
 		return false;
 
 	/*
@@ -290,7 +290,7 @@ clb_umwait(uint16_t port_id, uint16_t qidx, struct rte_mbuf **pkts __rte_unused,
 	/* this callback can't do more than one queue, omit multiqueue logic */
 	if (unlikely(nb_rx == 0)) {
 		queue_conf->n_empty_polls++;
-		if (unlikely(queue_conf->n_empty_polls > EMPTYPOLL_MAX)) {
+		if (unlikely(queue_conf->n_empty_polls > emptypoll_max)) {
 			struct rte_power_monitor_cond pmc;
 			int ret;
 
@@ -661,6 +661,18 @@ rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
 	return 0;
 }
 
+void
+rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max)
+{
+	emptypoll_max = max;
+}
+
+unsigned int
+rte_power_pmd_mgmt_get_emptypoll_max(void)
+{
+	return emptypoll_max;
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
 
@@ -669,4 +681,7 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 		struct pmd_core_cfg *cfg = &lcore_cfgs[i];
 		TAILQ_INIT(&cfg->head);
 	}
+
+	/* initialize config defaults */
+	emptypoll_max = 512;
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index 444e7b8a66..d5a94f8187 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -90,6 +90,33 @@ int
 rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
 		uint16_t port_id, uint16_t queue_id);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set a emptypoll_max to specified value. Used to specify the number of empty
+ * polls to wait before entering sleep state.
+ *
+ * @param max
+ *   The value to set emptypoll_max to.
+ */
+__rte_experimental
+void
+rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current value of emptypoll_max.
+ *
+ * @return
+ *   The current emptypoll_max value
+ */
+__rte_experimental
+unsigned int
+rte_power_pmd_mgmt_get_emptypoll_max(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 6ec6d5d96d..812843c3f3 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -38,4 +38,8 @@ EXPERIMENTAL {
 	# added in 21.02
 	rte_power_ethdev_pmgmt_queue_disable;
 	rte_power_ethdev_pmgmt_queue_enable;
+
+	# added in 22.07
+	rte_power_pmd_mgmt_get_emptypoll_max;
+	rte_power_pmd_mgmt_set_emptypoll_max;
 };
-- 
2.31.1

Re: [PATCH v3 1/4] lib/power: add get and set API for emptypoll max

[Ray Kinsella]

Kevin Laatz <kevin.laatz@intel.com> writes:

> Add new get/set APIs to configure emptypoll max which is used to
> determine when a queue can go into sleep state.
>
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> Acked-by: Ray Kinsella <mdr@ashroe.eu>
> ---
>  lib/power/rte_power_pmd_mgmt.c | 21 ++++++++++++++++++---
>  lib/power/rte_power_pmd_mgmt.h | 27 +++++++++++++++++++++++++++
>  lib/power/version.map          |  4 ++++
>  3 files changed, 49 insertions(+), 3 deletions(-)
>
Acked-by: Ray Kinsella <mdr@ashroe.eu>

[PATCH v3 2/4] lib/power: add get and set API for pause duration

[Kevin Laatz]

Add new get/set API for configuring 'pause_duration' which used to adjust
the pause mode callback duration.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Ray Kinsella <mdr@ashroe.eu>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

---
v3: changed printf to RTE_LOG
---
 lib/power/rte_power_pmd_mgmt.c | 25 +++++++++++++++++++++++--
 lib/power/rte_power_pmd_mgmt.h | 31 +++++++++++++++++++++++++++++++
 lib/power/version.map          |  2 ++
 3 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index dfb7ca9187..1374cc213d 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -12,6 +12,7 @@
 #include "rte_power_pmd_mgmt.h"
 
 unsigned int emptypoll_max;
+unsigned int pause_duration;
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -315,6 +316,7 @@ clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,
 	struct queue_list_entry *queue_conf = arg;
 	struct pmd_core_cfg *lcore_conf;
 	const bool empty = nb_rx == 0;
+	uint32_t pause_duration = rte_power_pmd_mgmt_get_pause_duration();
 
 	lcore_conf = &lcore_cfgs[lcore];
 
@@ -334,11 +336,11 @@ clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,
 		if (global_data.intrinsics_support.power_pause) {
 			const uint64_t cur = rte_rdtsc();
 			const uint64_t wait_tsc =
-					cur + global_data.tsc_per_us;
+					cur + global_data.tsc_per_us * pause_duration;
 			rte_power_pause(wait_tsc);
 		} else {
 			uint64_t i;
-			for (i = 0; i < global_data.pause_per_us; i++)
+			for (i = 0; i < global_data.pause_per_us * pause_duration; i++)
 				rte_pause();
 		}
 	}
@@ -673,6 +675,24 @@ rte_power_pmd_mgmt_get_emptypoll_max(void)
 	return emptypoll_max;
 }
 
+int
+rte_power_pmd_mgmt_set_pause_duration(unsigned int duration)
+{
+	if (duration == 0) {
+		RTE_LOG(ERR, POWER, "Pause duration must be greater than 0, value unchanged");
+		return -EINVAL;
+	}
+	pause_duration = duration;
+
+	return 0;
+}
+
+unsigned int
+rte_power_pmd_mgmt_get_pause_duration(void)
+{
+	return pause_duration;
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
 
@@ -684,4 +704,5 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 
 	/* initialize config defaults */
 	emptypoll_max = 512;
+	pause_duration = 1;
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index d5a94f8187..18a9c3abb5 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -117,6 +117,37 @@ __rte_experimental
 unsigned int
 rte_power_pmd_mgmt_get_emptypoll_max(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the pause_duration. Used to adjust the pause mode callback duration.
+ *
+ * @note Duration must be greater than zero.
+ *
+ * @param duration
+ *   The value to set pause_duration to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_pause_duration(unsigned int duration);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current value of pause_duration.
+ *
+ * @return
+ *   The current pause_duration value.
+ */
+__rte_experimental
+unsigned int
+rte_power_pmd_mgmt_get_pause_duration(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 812843c3f3..4673b719f9 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -41,5 +41,7 @@ EXPERIMENTAL {
 
 	# added in 22.07
 	rte_power_pmd_mgmt_get_emptypoll_max;
+	rte_power_pmd_mgmt_get_pause_duration;
 	rte_power_pmd_mgmt_set_emptypoll_max;
+	rte_power_pmd_mgmt_set_pause_duration;
 };
-- 
2.31.1

[PATCH v3 3/4] lib/power: add get and set API for scaling freq min and max with pstate mode

[Kevin Laatz]

Add new get/set API to allow the user or application to set the minimum
and maximum frequencies to use when scaling.
Previously, the frequency range was determined by the HW capabilities of
the CPU. With this new API, the user or application can constrain this
if required.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Ray Kinsella <mdr@ashroe.eu>

---
v3:
 * updated doxygen comments
 * added checks to ensure min/max value is valid
 * consider 0 as 'not set' for scaling_freq_max
---
 lib/power/power_pstate_cpufreq.c | 22 +++++++--
 lib/power/rte_power_pmd_mgmt.c   | 75 +++++++++++++++++++++++++++++
 lib/power/rte_power_pmd_mgmt.h   | 81 ++++++++++++++++++++++++++++++++
 lib/power/version.map            |  4 ++
 4 files changed, 177 insertions(+), 5 deletions(-)

diff --git a/lib/power/power_pstate_cpufreq.c b/lib/power/power_pstate_cpufreq.c
index f4c36179ec..db10deafe2 100644
--- a/lib/power/power_pstate_cpufreq.c
+++ b/lib/power/power_pstate_cpufreq.c
@@ -12,6 +12,7 @@
 
 #include <rte_memcpy.h>
 
+#include "rte_power_pmd_mgmt.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
 
@@ -354,6 +355,7 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	FILE *f_min = NULL, *f_max = NULL;
 	int ret = -1;
 	uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
+	int config_min_freq, config_max_freq;
 	uint32_t i, num_freqs = 0;
 
 	/* open all files */
@@ -388,6 +390,16 @@ power_get_available_freqs(struct pstate_power_info *pi)
 		goto out;
 	}
 
+	/* check for config set by user or application to limit frequency range */
+	config_min_freq = rte_power_pmd_mgmt_get_scaling_freq_min(pi->lcore_id);
+	if (config_min_freq < 0)
+		goto out;
+	config_max_freq = rte_power_pmd_mgmt_get_scaling_freq_max(pi->lcore_id);
+	if (config_max_freq < 0)
+		goto out;
+	sys_min_freq = RTE_MAX(sys_min_freq, (uint32_t)config_min_freq);
+	sys_max_freq = RTE_MIN(sys_max_freq, (uint32_t)config_max_freq);
+
 	if (sys_max_freq < sys_min_freq)
 		goto out;
 
@@ -411,8 +423,8 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	/* If turbo is available then there is one extra freq bucket
 	 * to store the sys max freq which value is base_max +1
 	 */
-	num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
-		pi->turbo_available;
+	num_freqs = (RTE_MIN(base_max_freq, sys_max_freq) - sys_min_freq) / BUS_FREQ
+			+ 1 + pi->turbo_available;
 	if (num_freqs >= RTE_MAX_LCORE_FREQS) {
 		RTE_LOG(ERR, POWER, "Too many available frequencies: %d\n",
 				num_freqs);
@@ -427,10 +439,10 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	 */
 	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 		if ((i == 0) && pi->turbo_available)
-			pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
+			pi->freqs[pi->nb_freqs++] = RTE_MIN(base_max_freq, sys_max_freq) + 1;
 		else
-			pi->freqs[pi->nb_freqs++] =
-			base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
+			pi->freqs[pi->nb_freqs++] = RTE_MIN(base_max_freq, sys_max_freq) -
+					(i - pi->turbo_available) * BUS_FREQ;
 	}
 
 	ret = 0;
diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index 1374cc213d..df6d3e8e15 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -10,9 +10,12 @@
 #include <rte_power_intrinsics.h>
 
 #include "rte_power_pmd_mgmt.h"
+#include "power_common.h"
 
 unsigned int emptypoll_max;
 unsigned int pause_duration;
+unsigned int scale_freq_min[RTE_MAX_LCORE];
+unsigned int scale_freq_max[RTE_MAX_LCORE];
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -693,8 +696,75 @@ rte_power_pmd_mgmt_get_pause_duration(void)
 	return pause_duration;
 }
 
+int
+rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	if (min > scale_freq_max[lcore]) {
+		RTE_LOG(ERR, POWER, "Invalid min frequency: Cannot be greater than max frequency");
+		return -EINVAL;
+	}
+	scale_freq_min[lcore] = min;
+
+	return 0;
+}
+
+int
+rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	/* Zero means 'not set'. Use UINT32_MAX to enable RTE_MIN/MAX macro use when scaling. */
+	if (max == 0)
+		max = UINT32_MAX;
+	if (max < scale_freq_min[lcore]) {
+		RTE_LOG(ERR, POWER, "Invalid max frequency: Cannot be less than min frequency");
+		return -EINVAL;
+	}
+
+	scale_freq_max[lcore] = max;
+
+	return 0;
+}
+
+int
+rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	if (scale_freq_max[lcore] == 0)
+		RTE_LOG(DEBUG, POWER, "Scaling freq min config not set. Using sysfs min freq.\n");
+
+	return scale_freq_min[lcore];
+}
+
+int
+rte_power_pmd_mgmt_get_scaling_freq_max(unsigned int lcore)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	if (scale_freq_max[lcore] == UINT32_MAX)
+		RTE_LOG(DEBUG, POWER, "Scaling freq max config not set. Using sysfs max freq.\n");
+
+	return scale_freq_max[lcore];
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
+	int j;
 
 	/* initialize all tailqs */
 	for (i = 0; i < RTE_DIM(lcore_cfgs); i++) {
@@ -705,4 +775,9 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	/* initialize config defaults */
 	emptypoll_max = 512;
 	pause_duration = 1;
+	/* scaling defaults out of range to ensure not used unless set by user or app */
+	for (j = 0; j < RTE_MAX_LCORE; j++) {
+		scale_freq_min[j] = 0;
+		scale_freq_max[j] = UINT32_MAX;
+	}
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index 18a9c3abb5..9ca75b91c6 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -148,6 +148,87 @@ __rte_experimental
 unsigned int
 rte_power_pmd_mgmt_get_pause_duration(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the min frequency to be used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to set the min frequency for.
+ * @param min
+ *   The value, in KiloHertz, to set the minimum frequency to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the max frequency to be used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to set the max frequency for.
+ * @param max
+ *   The value, in KiloHertz, to set the maximum frequency to.
+ *   If 'max' is 0, it is considered 'not set'.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current configured min frequency used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to get the min frequency for.
+ * @return
+ *   0 if no value has been configured via the 'set' API.
+ *   >0 if a minimum frequency has been configured. Value is the minimum frequency
+ *   , in KiloHertz, used for frequency scaling.
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current configured max frequency used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to get the max frequency for.
+ * @return
+ *   UINT32_MAX if no value has been configured via the 'set' API.
+ *   On success, the current configured maximum frequency, in KiloHertz, used for
+ *   frequency scaling.
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_get_scaling_freq_max(unsigned int lcore);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 4673b719f9..a687754f4a 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -42,6 +42,10 @@ EXPERIMENTAL {
 	# added in 22.07
 	rte_power_pmd_mgmt_get_emptypoll_max;
 	rte_power_pmd_mgmt_get_pause_duration;
+	rte_power_pmd_mgmt_get_scaling_freq_max;
+	rte_power_pmd_mgmt_get_scaling_freq_min;
 	rte_power_pmd_mgmt_set_emptypoll_max;
 	rte_power_pmd_mgmt_set_pause_duration;
+	rte_power_pmd_mgmt_set_scaling_freq_max;
+	rte_power_pmd_mgmt_set_scaling_freq_min;
 };
-- 
2.31.1

Re: [PATCH v3 3/4] lib/power: add get and set API for scaling freq min and max with pstate mode

[Burakov, Anatoly]

On 23-May-22 9:21 PM, Kevin Laatz wrote:
> Add new get/set API to allow the user or application to set the minimum
> and maximum frequencies to use when scaling.
> Previously, the frequency range was determined by the HW capabilities of
> the CPU. With this new API, the user or application can constrain this
> if required.
> 
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> Acked-by: Ray Kinsella <mdr@ashroe.eu>
> 
> ---

<snip>

> +int
> +rte_power_pmd_mgmt_get_scaling_freq_max(unsigned int lcore)
> +{
> +	if (lcore >= RTE_MAX_LCORE) {
> +		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
> +		return -EINVAL;
> +	}
> +
> +	if (scale_freq_max[lcore] == UINT32_MAX)
> +		RTE_LOG(DEBUG, POWER, "Scaling freq max config not set. Using sysfs max freq.\n");
> +
> +	return scale_freq_max[lcore];

If you're using 0 as "not set" you should return 0 when it's not set :)

> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
> + *
> + * Set the min frequency to be used for frequency scaling.

...or zero to use defaults?

> + *
> + * @note Supported by: Pstate mode.
> + *
> + * @param lcore
> + *   The ID of the lcore to set the min frequency for.
> + * @param min
> + *   The value, in KiloHertz, to set the minimum frequency to.
> + * @return
> + *   0 on success
> + *   <0 on error
> + */
> +__rte_experimental
> +int
> +rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
> + *
> + * Set the max frequency to be used for frequency scaling.

...or zero to use defaults?

> + *
> + * @note Supported by: Pstate mode.
> + *
> + * @param lcore
> + *   The ID of the lcore to set the max frequency for.
> + * @param max
> + *   The value, in KiloHertz, to set the maximum frequency to.
> + *   If 'max' is 0, it is considered 'not set'.
> + * @return
> + *   0 on success
> + *   <0 on error
> + */
> +__rte_experimental
> +int
> +rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
> + *
> + * Get the current configured min frequency used for frequency scaling.
> + *
> + * @note Supported by: Pstate mode.
> + *
> + * @param lcore
> + *   The ID of the lcore to get the min frequency for.
> + * @return
> + *   0 if no value has been configured via the 'set' API.
> + *   >0 if a minimum frequency has been configured. Value is the minimum frequency
> + *   , in KiloHertz, used for frequency scaling.
> + *   <0 on error
> + */
> +__rte_experimental
> +int
> +rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
> + *
> + * Get the current configured max frequency used for frequency scaling.
> + *
> + * @note Supported by: Pstate mode.
> + *
> + * @param lcore
> + *   The ID of the lcore to get the max frequency for.
> + * @return
> + *   UINT32_MAX if no value has been configured via the 'set' API.

The comment now mismatches the code.

Once the above is fixed,

Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

-- 
Thanks,
Anatoly

[PATCH v3 4/4] examples/l3fwd_power: add cli for configurable options

[Kevin Laatz]

Add CLI options to l3fwd_power to utilize the new power APIs introduced in
this patchset. These CLI options allow the user to configure the
heuritstics made available through the new API via the l3fwd_power
application options.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>

---
v2: add doc update for l3fwd-power
v3: move setters out of arg parsing
---
 .../sample_app_ug/l3_forward_power_man.rst    |  8 ++
 examples/l3fwd-power/main.c                   | 86 ++++++++++++++++++-
 2 files changed, 93 insertions(+), 1 deletion(-)

diff --git a/doc/guides/sample_app_ug/l3_forward_power_man.rst b/doc/guides/sample_app_ug/l3_forward_power_man.rst
index 2e350c45f1..8f6d906200 100644
--- a/doc/guides/sample_app_ug/l3_forward_power_man.rst
+++ b/doc/guides/sample_app_ug/l3_forward_power_man.rst
@@ -109,6 +109,14 @@ where,
 
 *   --pmd-mgmt: PMD power management mode.
 
+*   --max-empty-polls : Number of empty polls to wait before entering sleep state. Applies to --pmd-mgmt mode only.
+
+*   --pause-duration: Set the duration of the pause callback (microseconds). Applies to --pmd-mgmt mode only.
+
+*   --scale-freq-min: Set minimum frequency for scaling. Applies to --pmd-mgmt mode only.
+
+*   --scale-freq-max: Set maximum frequency for scaling. Applies to --pmd-mgmt mode only.
+
 See :doc:`l3_forward` for details.
 The L3fwd-power example reuses the L3fwd command line options.
 
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 20e5b59af9..887c6eae3f 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -265,6 +265,10 @@ static struct rte_eth_conf port_conf = {
 };
 
 static uint32_t max_pkt_len;
+static uint32_t max_empty_polls = 512;
+static uint32_t pause_duration = 1;
+static uint32_t scale_freq_min;
+static uint32_t scale_freq_max;
 
 static struct rte_mempool * pktmbuf_pool[NB_SOCKETS];
 
@@ -1626,10 +1630,32 @@ print_usage(const char *prgname)
 		" empty polls, full polls, and core busyness to telemetry\n"
 		" --interrupt-only: enable interrupt-only mode\n"
 		" --pmd-mgmt MODE: enable PMD power management mode. "
-		"Currently supported modes: baseline, monitor, pause, scale\n",
+		"Currently supported modes: baseline, monitor, pause, scale\n"
+		"  --max-empty-polls MAX_EMPTY_POLLS: number of empty polls to"
+		" wait before entering sleep state\n"
+		"  --pause-duration DURATION: set the duration, in microseconds,"
+		" of the pause callback\n"
+		"  --scale-freq-min FREQ_MIN: set minimum frequency for scaling mode for"
+		" all application lcores (FREQ_MIN must be in kHz, in increments of 100MHz)\n"
+		"  --scale-freq-max FREQ_MAX: set maximum frequency for scaling mode for"
+		" all application lcores (FREQ_MAX must be in kHz, in increments of 100MHz)\n",
 		prgname);
 }
 
+static int
+parse_int(const char *opt)
+{
+	char *end = NULL;
+	unsigned long val;
+
+	/* parse integer string */
+	val = strtoul(opt, &end, 10);
+	if ((opt[0] == '\0') || (end == NULL) || (*end != '\0'))
+		return -1;
+
+	return val;
+}
+
 static int parse_max_pkt_len(const char *pktlen)
 {
 	char *end = NULL;
@@ -1803,6 +1829,10 @@ parse_ep_config(const char *q_arg)
 #define CMD_LINE_OPT_TELEMETRY "telemetry"
 #define CMD_LINE_OPT_PMD_MGMT "pmd-mgmt"
 #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len"
+#define CMD_LINE_OPT_MAX_EMPTY_POLLS "max-empty-polls"
+#define CMD_LINE_OPT_PAUSE_DURATION "pause-duration"
+#define CMD_LINE_OPT_SCALE_FREQ_MIN "scale-freq-min"
+#define CMD_LINE_OPT_SCALE_FREQ_MAX "scale-freq-max"
 
 /* Parse the argument given in the command line of the application */
 static int
@@ -1825,6 +1855,10 @@ parse_args(int argc, char **argv)
 		{CMD_LINE_OPT_TELEMETRY, 0, 0, 0},
 		{CMD_LINE_OPT_INTERRUPT_ONLY, 0, 0, 0},
 		{CMD_LINE_OPT_PMD_MGMT, 1, 0, 0},
+		{CMD_LINE_OPT_MAX_EMPTY_POLLS, 1, 0, 0},
+		{CMD_LINE_OPT_PAUSE_DURATION, 1, 0, 0},
+		{CMD_LINE_OPT_SCALE_FREQ_MIN, 1, 0, 0},
+		{CMD_LINE_OPT_SCALE_FREQ_MAX, 1, 0, 0},
 		{NULL, 0, 0, 0}
 	};
 
@@ -1975,6 +2009,34 @@ parse_args(int argc, char **argv)
 				parse_ptype = 1;
 			}
 
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_MAX_EMPTY_POLLS,
+					sizeof(CMD_LINE_OPT_MAX_EMPTY_POLLS))) {
+				printf("Maximum empty polls configured\n");
+				max_empty_polls = parse_int(optarg);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_PAUSE_DURATION,
+					sizeof(CMD_LINE_OPT_PAUSE_DURATION))) {
+				printf("Pause duration configured\n");
+				pause_duration = parse_int(optarg);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_SCALE_FREQ_MIN,
+					sizeof(CMD_LINE_OPT_SCALE_FREQ_MIN))) {
+				printf("Scaling frequency minimum configured\n");
+				scale_freq_min = parse_int(optarg);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_SCALE_FREQ_MAX,
+					sizeof(CMD_LINE_OPT_SCALE_FREQ_MAX))) {
+				printf("Scaling frequency maximum configured\n");
+				scale_freq_max = parse_int(optarg);
+			}
+
 			break;
 
 		default:
@@ -2801,6 +2863,28 @@ main(int argc, char **argv)
 			}
 
 			if (app_mode == APP_MODE_PMD_MGMT && !baseline_enabled) {
+				/* Set power_pmd_mgmt configs passed by user */
+				rte_power_pmd_mgmt_set_emptypoll_max(max_empty_polls);
+				ret = rte_power_pmd_mgmt_set_pause_duration(pause_duration);
+				if (ret < 0)
+					rte_exit(EXIT_FAILURE,
+						"Error setting pause_duration: err=%d, lcore=%d\n",
+							ret, lcore_id);
+
+				ret = rte_power_pmd_mgmt_set_scaling_freq_min(lcore_id,
+						scale_freq_min);
+				if (ret < 0)
+					rte_exit(EXIT_FAILURE,
+						"Error setting scaling freq min: err=%d, lcore=%d\n",
+							ret, lcore_id);
+
+				ret = rte_power_pmd_mgmt_set_scaling_freq_max(lcore_id,
+						scale_freq_max);
+				if (ret < 0)
+					rte_exit(EXIT_FAILURE,
+						"Error setting scaling freq max: err=%d, lcore %d\n",
+							ret, lcore_id);
+
 				ret = rte_power_ethdev_pmgmt_queue_enable(
 						lcore_id, portid, queueid,
 						pmgmt_type);
-- 
2.31.1

Re: [PATCH v3 4/4] examples/l3fwd_power: add cli for configurable options

[Burakov, Anatoly]

On 23-May-22 9:21 PM, Kevin Laatz wrote:
> Add CLI options to l3fwd_power to utilize the new power APIs introduced in
> this patchset. These CLI options allow the user to configure the
> heuritstics made available through the new API via the l3fwd_power
> application options.
> 
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> 
> ---
> v2: add doc update for l3fwd-power
> v3: move setters out of arg parsing
> ---

Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

-- 
Thanks,
Anatoly

[PATCH v4 0/4] Add APIs for configurable power options

[Kevin Laatz]

The power library contains some variables which are currently set by
defines, hard-coded values or set using sysfs values. In order to
configure these, code changes and recompiles are required, making
configuring these variables tedious.

This patchset introduces some new get/set APIs which allow users and
applications to configure there settings to suit their use-cases.
In addition, CLI options have been added to l3fwd_power to demonstrate
how an application could use these APIs to expose the options to users
without needing code changes to configure them.

---
v4:
* fix return value when scaling_freq_max is not set
* fix mismatching comments

v3:
* move setters from arg parse function to init
* consider 0 as 'not set' for scaling_freq_max
* other minor fixes

v2:
* add doc update for l3fwd-power
* order version.map additions alphabetically

Kevin Laatz (4):
  lib/power: add get and set API for emptypoll max
  lib/power: add get and set API for pause duration
  lib/power: add get and set API for scaling freq min and max with
    pstate mode
  examples/l3fwd_power: add cli for configurable options

 .../sample_app_ug/l3_forward_power_man.rst    |   8 +
 examples/l3fwd-power/main.c                   |  86 ++++++++++-
 lib/power/power_pstate_cpufreq.c              |  24 ++-
 lib/power/rte_power_pmd_mgmt.c                | 123 +++++++++++++++-
 lib/power/rte_power_pmd_mgmt.h                | 139 ++++++++++++++++++
 lib/power/version.map                         |  10 ++
 6 files changed, 379 insertions(+), 11 deletions(-)

-- 
2.31.1

[PATCH v4 1/4] lib/power: add get and set API for emptypoll max

[Kevin Laatz]

Add new get/set APIs to configure emptypoll max which is used to
determine when a queue can go into sleep state.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Ray Kinsella <mdr@ashroe.eu>
---
 lib/power/rte_power_pmd_mgmt.c | 21 ++++++++++++++++++---
 lib/power/rte_power_pmd_mgmt.h | 27 +++++++++++++++++++++++++++
 lib/power/version.map          |  4 ++++
 3 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index 39a2b4cd23..dfb7ca9187 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -11,7 +11,7 @@
 
 #include "rte_power_pmd_mgmt.h"
 
-#define EMPTYPOLL_MAX  512
+unsigned int emptypoll_max;
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -206,7 +206,7 @@ queue_can_sleep(struct pmd_core_cfg *cfg, struct queue_list_entry *qcfg)
 	qcfg->n_empty_polls++;
 
 	/* if we haven't reached threshold for empty polls, we can't sleep */
-	if (qcfg->n_empty_polls <= EMPTYPOLL_MAX)
+	if (qcfg->n_empty_polls <= emptypoll_max)
 		return false;
 
 	/*
@@ -290,7 +290,7 @@ clb_umwait(uint16_t port_id, uint16_t qidx, struct rte_mbuf **pkts __rte_unused,
 	/* this callback can't do more than one queue, omit multiqueue logic */
 	if (unlikely(nb_rx == 0)) {
 		queue_conf->n_empty_polls++;
-		if (unlikely(queue_conf->n_empty_polls > EMPTYPOLL_MAX)) {
+		if (unlikely(queue_conf->n_empty_polls > emptypoll_max)) {
 			struct rte_power_monitor_cond pmc;
 			int ret;
 
@@ -661,6 +661,18 @@ rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
 	return 0;
 }
 
+void
+rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max)
+{
+	emptypoll_max = max;
+}
+
+unsigned int
+rte_power_pmd_mgmt_get_emptypoll_max(void)
+{
+	return emptypoll_max;
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
 
@@ -669,4 +681,7 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 		struct pmd_core_cfg *cfg = &lcore_cfgs[i];
 		TAILQ_INIT(&cfg->head);
 	}
+
+	/* initialize config defaults */
+	emptypoll_max = 512;
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index 444e7b8a66..d5a94f8187 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -90,6 +90,33 @@ int
 rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
 		uint16_t port_id, uint16_t queue_id);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set a emptypoll_max to specified value. Used to specify the number of empty
+ * polls to wait before entering sleep state.
+ *
+ * @param max
+ *   The value to set emptypoll_max to.
+ */
+__rte_experimental
+void
+rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current value of emptypoll_max.
+ *
+ * @return
+ *   The current emptypoll_max value
+ */
+__rte_experimental
+unsigned int
+rte_power_pmd_mgmt_get_emptypoll_max(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 6ec6d5d96d..812843c3f3 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -38,4 +38,8 @@ EXPERIMENTAL {
 	# added in 21.02
 	rte_power_ethdev_pmgmt_queue_disable;
 	rte_power_ethdev_pmgmt_queue_enable;
+
+	# added in 22.07
+	rte_power_pmd_mgmt_get_emptypoll_max;
+	rte_power_pmd_mgmt_set_emptypoll_max;
 };
-- 
2.31.1

Re: [PATCH v4 1/4] lib/power: add get and set API for emptypoll max

[David Hunt]

On 24/05/2022 14:14, Kevin Laatz wrote:
> Add new get/set APIs to configure emptypoll max which is used to
> determine when a queue can go into sleep state.
>
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> Acked-by: Ray Kinsella <mdr@ashroe.eu>
> ---
>   lib/power/rte_power_pmd_mgmt.c | 21 ++++++++++++++++++---
>   lib/power/rte_power_pmd_mgmt.h | 27 +++++++++++++++++++++++++++
>   lib/power/version.map          |  4 ++++
>   3 files changed, 49 insertions(+), 3 deletions(-)
>

Tested-by: David Hunt <david.hunt@intel.com>

[PATCH v4 2/4] lib/power: add get and set API for pause duration

[Kevin Laatz]

Add new get/set API for configuring 'pause_duration' which used to adjust
the pause mode callback duration.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Ray Kinsella <mdr@ashroe.eu>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

---
v3: changed printf to RTE_LOG
---
 lib/power/rte_power_pmd_mgmt.c | 25 +++++++++++++++++++++++--
 lib/power/rte_power_pmd_mgmt.h | 31 +++++++++++++++++++++++++++++++
 lib/power/version.map          |  2 ++
 3 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index dfb7ca9187..1374cc213d 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -12,6 +12,7 @@
 #include "rte_power_pmd_mgmt.h"
 
 unsigned int emptypoll_max;
+unsigned int pause_duration;
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -315,6 +316,7 @@ clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,
 	struct queue_list_entry *queue_conf = arg;
 	struct pmd_core_cfg *lcore_conf;
 	const bool empty = nb_rx == 0;
+	uint32_t pause_duration = rte_power_pmd_mgmt_get_pause_duration();
 
 	lcore_conf = &lcore_cfgs[lcore];
 
@@ -334,11 +336,11 @@ clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,
 		if (global_data.intrinsics_support.power_pause) {
 			const uint64_t cur = rte_rdtsc();
 			const uint64_t wait_tsc =
-					cur + global_data.tsc_per_us;
+					cur + global_data.tsc_per_us * pause_duration;
 			rte_power_pause(wait_tsc);
 		} else {
 			uint64_t i;
-			for (i = 0; i < global_data.pause_per_us; i++)
+			for (i = 0; i < global_data.pause_per_us * pause_duration; i++)
 				rte_pause();
 		}
 	}
@@ -673,6 +675,24 @@ rte_power_pmd_mgmt_get_emptypoll_max(void)
 	return emptypoll_max;
 }
 
+int
+rte_power_pmd_mgmt_set_pause_duration(unsigned int duration)
+{
+	if (duration == 0) {
+		RTE_LOG(ERR, POWER, "Pause duration must be greater than 0, value unchanged");
+		return -EINVAL;
+	}
+	pause_duration = duration;
+
+	return 0;
+}
+
+unsigned int
+rte_power_pmd_mgmt_get_pause_duration(void)
+{
+	return pause_duration;
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
 
@@ -684,4 +704,5 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 
 	/* initialize config defaults */
 	emptypoll_max = 512;
+	pause_duration = 1;
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index d5a94f8187..18a9c3abb5 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -117,6 +117,37 @@ __rte_experimental
 unsigned int
 rte_power_pmd_mgmt_get_emptypoll_max(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the pause_duration. Used to adjust the pause mode callback duration.
+ *
+ * @note Duration must be greater than zero.
+ *
+ * @param duration
+ *   The value to set pause_duration to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_pause_duration(unsigned int duration);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current value of pause_duration.
+ *
+ * @return
+ *   The current pause_duration value.
+ */
+__rte_experimental
+unsigned int
+rte_power_pmd_mgmt_get_pause_duration(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 812843c3f3..4673b719f9 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -41,5 +41,7 @@ EXPERIMENTAL {
 
 	# added in 22.07
 	rte_power_pmd_mgmt_get_emptypoll_max;
+	rte_power_pmd_mgmt_get_pause_duration;
 	rte_power_pmd_mgmt_set_emptypoll_max;
+	rte_power_pmd_mgmt_set_pause_duration;
 };
-- 
2.31.1

Re: [PATCH v4 2/4] lib/power: add get and set API for pause duration

[David Hunt]

On 24/05/2022 14:14, Kevin Laatz wrote:
> Add new get/set API for configuring 'pause_duration' which used to adjust
> the pause mode callback duration.
>
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> Acked-by: Ray Kinsella <mdr@ashroe.eu>
> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
>
> ---
> v3: changed printf to RTE_LOG
> ---
>   lib/power/rte_power_pmd_mgmt.c | 25 +++++++++++++++++++++++--
>   lib/power/rte_power_pmd_mgmt.h | 31 +++++++++++++++++++++++++++++++
>   lib/power/version.map          |  2 ++
>   3 files changed, 56 insertions(+), 2 deletions(-)
>

Tested-by: David Hunt <david.hunt@intel.com>

[PATCH v4 3/4] lib/power: add get and set API for scaling freq min and max with pstate mode

[Kevin Laatz]

Add new get/set API to allow the user or application to set the minimum
and maximum frequencies to use when scaling.
Previously, the frequency range was determined by the HW capabilities of
the CPU. With this new API, the user or application can constrain this
if required.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Ray Kinsella <mdr@ashroe.eu>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

---
v4:
 * fix mismatch between comments and code
 * fix return value when max freq is not set

v3:
 * updated doxygen comments
 * added checks to ensure min/max value is valid
 * consider 0 as 'not set' for scaling_freq_max
---
 lib/power/power_pstate_cpufreq.c | 24 ++++++++--
 lib/power/rte_power_pmd_mgmt.c   | 77 ++++++++++++++++++++++++++++++
 lib/power/rte_power_pmd_mgmt.h   | 81 ++++++++++++++++++++++++++++++++
 lib/power/version.map            |  4 ++
 4 files changed, 181 insertions(+), 5 deletions(-)

diff --git a/lib/power/power_pstate_cpufreq.c b/lib/power/power_pstate_cpufreq.c
index f4c36179ec..78c9197695 100644
--- a/lib/power/power_pstate_cpufreq.c
+++ b/lib/power/power_pstate_cpufreq.c
@@ -12,6 +12,7 @@
 
 #include <rte_memcpy.h>
 
+#include "rte_power_pmd_mgmt.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
 
@@ -354,6 +355,7 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	FILE *f_min = NULL, *f_max = NULL;
 	int ret = -1;
 	uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
+	int config_min_freq, config_max_freq;
 	uint32_t i, num_freqs = 0;
 
 	/* open all files */
@@ -388,6 +390,18 @@ power_get_available_freqs(struct pstate_power_info *pi)
 		goto out;
 	}
 
+	/* check for config set by user or application to limit frequency range */
+	config_min_freq = rte_power_pmd_mgmt_get_scaling_freq_min(pi->lcore_id);
+	if (config_min_freq < 0)
+		goto out;
+	config_max_freq = rte_power_pmd_mgmt_get_scaling_freq_max(pi->lcore_id);
+	if (config_max_freq < 0)
+		goto out;
+
+	sys_min_freq = RTE_MAX(sys_min_freq, (uint32_t)config_min_freq);
+	if (config_max_freq > 0) /* Only use config_max_freq if a value has been set */
+		sys_max_freq = RTE_MIN(sys_max_freq, (uint32_t)config_max_freq);
+
 	if (sys_max_freq < sys_min_freq)
 		goto out;
 
@@ -411,8 +425,8 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	/* If turbo is available then there is one extra freq bucket
 	 * to store the sys max freq which value is base_max +1
 	 */
-	num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
-		pi->turbo_available;
+	num_freqs = (RTE_MIN(base_max_freq, sys_max_freq) - sys_min_freq) / BUS_FREQ
+			+ 1 + pi->turbo_available;
 	if (num_freqs >= RTE_MAX_LCORE_FREQS) {
 		RTE_LOG(ERR, POWER, "Too many available frequencies: %d\n",
 				num_freqs);
@@ -427,10 +441,10 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	 */
 	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 		if ((i == 0) && pi->turbo_available)
-			pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
+			pi->freqs[pi->nb_freqs++] = RTE_MIN(base_max_freq, sys_max_freq) + 1;
 		else
-			pi->freqs[pi->nb_freqs++] =
-			base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
+			pi->freqs[pi->nb_freqs++] = RTE_MIN(base_max_freq, sys_max_freq) -
+					(i - pi->turbo_available) * BUS_FREQ;
 	}
 
 	ret = 0;
diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index 1374cc213d..a66e08e574 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -10,9 +10,12 @@
 #include <rte_power_intrinsics.h>
 
 #include "rte_power_pmd_mgmt.h"
+#include "power_common.h"
 
 unsigned int emptypoll_max;
 unsigned int pause_duration;
+unsigned int scale_freq_min[RTE_MAX_LCORE];
+unsigned int scale_freq_max[RTE_MAX_LCORE];
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -693,8 +696,77 @@ rte_power_pmd_mgmt_get_pause_duration(void)
 	return pause_duration;
 }
 
+int
+rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	if (min > scale_freq_max[lcore]) {
+		RTE_LOG(ERR, POWER, "Invalid min frequency: Cannot be greater than max frequency");
+		return -EINVAL;
+	}
+	scale_freq_min[lcore] = min;
+
+	return 0;
+}
+
+int
+rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	/* Zero means 'not set'. Use UINT32_MAX to enable RTE_MIN/MAX macro use when scaling. */
+	if (max == 0)
+		max = UINT32_MAX;
+	if (max < scale_freq_min[lcore]) {
+		RTE_LOG(ERR, POWER, "Invalid max frequency: Cannot be less than min frequency");
+		return -EINVAL;
+	}
+
+	scale_freq_max[lcore] = max;
+
+	return 0;
+}
+
+int
+rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	if (scale_freq_max[lcore] == 0)
+		RTE_LOG(DEBUG, POWER, "Scaling freq min config not set. Using sysfs min freq.\n");
+
+	return scale_freq_min[lcore];
+}
+
+int
+rte_power_pmd_mgmt_get_scaling_freq_max(unsigned int lcore)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	if (scale_freq_max[lcore] == UINT32_MAX) {
+		RTE_LOG(DEBUG, POWER, "Scaling freq max config not set. Using sysfs max freq.\n");
+		return 0;
+	}
+
+	return scale_freq_max[lcore];
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
+	int j;
 
 	/* initialize all tailqs */
 	for (i = 0; i < RTE_DIM(lcore_cfgs); i++) {
@@ -705,4 +777,9 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	/* initialize config defaults */
 	emptypoll_max = 512;
 	pause_duration = 1;
+	/* scaling defaults out of range to ensure not used unless set by user or app */
+	for (j = 0; j < RTE_MAX_LCORE; j++) {
+		scale_freq_min[j] = 0;
+		scale_freq_max[j] = UINT32_MAX;
+	}
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index 18a9c3abb5..789fbe0e4d 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -148,6 +148,87 @@ __rte_experimental
 unsigned int
 rte_power_pmd_mgmt_get_pause_duration(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the min frequency to be used for frequency scaling or zero to use defaults.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to set the min frequency for.
+ * @param min
+ *   The value, in KiloHertz, to set the minimum frequency to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the max frequency to be used for frequency scaling or zero to use defaults.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to set the max frequency for.
+ * @param max
+ *   The value, in KiloHertz, to set the maximum frequency to.
+ *   If 'max' is 0, it is considered 'not set'.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current configured min frequency used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to get the min frequency for.
+ * @return
+ *   0 if no value has been configured via the 'set' API.
+ *   >0 if a minimum frequency has been configured. Value is the minimum frequency
+ *   , in KiloHertz, used for frequency scaling.
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current configured max frequency used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to get the max frequency for.
+ * @return
+ *   0 if no value has been configured via the 'set' API.
+ *   >0 if a maximum frequency has been configured. Value is the maximum frequency
+ *   , in KiloHertz, used for frequency scaling.
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_get_scaling_freq_max(unsigned int lcore);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 4673b719f9..a687754f4a 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -42,6 +42,10 @@ EXPERIMENTAL {
 	# added in 22.07
 	rte_power_pmd_mgmt_get_emptypoll_max;
 	rte_power_pmd_mgmt_get_pause_duration;
+	rte_power_pmd_mgmt_get_scaling_freq_max;
+	rte_power_pmd_mgmt_get_scaling_freq_min;
 	rte_power_pmd_mgmt_set_emptypoll_max;
 	rte_power_pmd_mgmt_set_pause_duration;
+	rte_power_pmd_mgmt_set_scaling_freq_max;
+	rte_power_pmd_mgmt_set_scaling_freq_min;
 };
-- 
2.31.1

Re: [PATCH v4 3/4] lib/power: add get and set API for scaling freq min and max with pstate mode

[David Hunt]

On 24/05/2022 14:14, Kevin Laatz wrote:
> Add new get/set API to allow the user or application to set the minimum
> and maximum frequencies to use when scaling.
> Previously, the frequency range was determined by the HW capabilities of
> the CPU. With this new API, the user or application can constrain this
> if required.
>
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> Acked-by: Ray Kinsella <mdr@ashroe.eu>
> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
>
> ---
> v4:
>   * fix mismatch between comments and code
>   * fix return value when max freq is not set
>
> v3:
>   * updated doxygen comments
>   * added checks to ensure min/max value is valid
>   * consider 0 as 'not set' for scaling_freq_max
> ---
>   lib/power/power_pstate_cpufreq.c | 24 ++++++++--
>   lib/power/rte_power_pmd_mgmt.c   | 77 ++++++++++++++++++++++++++++++
>   lib/power/rte_power_pmd_mgmt.h   | 81 ++++++++++++++++++++++++++++++++
>   lib/power/version.map            |  4 ++
>   4 files changed, 181 insertions(+), 5 deletions(-)
>

Tested-by: David Hunt <david.hunt@intel.com>

[PATCH v4 4/4] examples/l3fwd_power: add cli for configurable options

[Kevin Laatz]

Add CLI options to l3fwd_power to utilize the new power APIs introduced in
this patchset. These CLI options allow the user to configure the
heuritstics made available through the new API via the l3fwd_power
application options.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

---
v3: move setters out of arg parsing
v2: add doc update for l3fwd-power
---
 .../sample_app_ug/l3_forward_power_man.rst    |  8 ++
 examples/l3fwd-power/main.c                   | 86 ++++++++++++++++++-
 2 files changed, 93 insertions(+), 1 deletion(-)

diff --git a/doc/guides/sample_app_ug/l3_forward_power_man.rst b/doc/guides/sample_app_ug/l3_forward_power_man.rst
index 2e350c45f1..8f6d906200 100644
--- a/doc/guides/sample_app_ug/l3_forward_power_man.rst
+++ b/doc/guides/sample_app_ug/l3_forward_power_man.rst
@@ -109,6 +109,14 @@ where,
 
 *   --pmd-mgmt: PMD power management mode.
 
+*   --max-empty-polls : Number of empty polls to wait before entering sleep state. Applies to --pmd-mgmt mode only.
+
+*   --pause-duration: Set the duration of the pause callback (microseconds). Applies to --pmd-mgmt mode only.
+
+*   --scale-freq-min: Set minimum frequency for scaling. Applies to --pmd-mgmt mode only.
+
+*   --scale-freq-max: Set maximum frequency for scaling. Applies to --pmd-mgmt mode only.
+
 See :doc:`l3_forward` for details.
 The L3fwd-power example reuses the L3fwd command line options.
 
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 20e5b59af9..887c6eae3f 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -265,6 +265,10 @@ static struct rte_eth_conf port_conf = {
 };
 
 static uint32_t max_pkt_len;
+static uint32_t max_empty_polls = 512;
+static uint32_t pause_duration = 1;
+static uint32_t scale_freq_min;
+static uint32_t scale_freq_max;
 
 static struct rte_mempool * pktmbuf_pool[NB_SOCKETS];
 
@@ -1626,10 +1630,32 @@ print_usage(const char *prgname)
 		" empty polls, full polls, and core busyness to telemetry\n"
 		" --interrupt-only: enable interrupt-only mode\n"
 		" --pmd-mgmt MODE: enable PMD power management mode. "
-		"Currently supported modes: baseline, monitor, pause, scale\n",
+		"Currently supported modes: baseline, monitor, pause, scale\n"
+		"  --max-empty-polls MAX_EMPTY_POLLS: number of empty polls to"
+		" wait before entering sleep state\n"
+		"  --pause-duration DURATION: set the duration, in microseconds,"
+		" of the pause callback\n"
+		"  --scale-freq-min FREQ_MIN: set minimum frequency for scaling mode for"
+		" all application lcores (FREQ_MIN must be in kHz, in increments of 100MHz)\n"
+		"  --scale-freq-max FREQ_MAX: set maximum frequency for scaling mode for"
+		" all application lcores (FREQ_MAX must be in kHz, in increments of 100MHz)\n",
 		prgname);
 }
 
+static int
+parse_int(const char *opt)
+{
+	char *end = NULL;
+	unsigned long val;
+
+	/* parse integer string */
+	val = strtoul(opt, &end, 10);
+	if ((opt[0] == '\0') || (end == NULL) || (*end != '\0'))
+		return -1;
+
+	return val;
+}
+
 static int parse_max_pkt_len(const char *pktlen)
 {
 	char *end = NULL;
@@ -1803,6 +1829,10 @@ parse_ep_config(const char *q_arg)
 #define CMD_LINE_OPT_TELEMETRY "telemetry"
 #define CMD_LINE_OPT_PMD_MGMT "pmd-mgmt"
 #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len"
+#define CMD_LINE_OPT_MAX_EMPTY_POLLS "max-empty-polls"
+#define CMD_LINE_OPT_PAUSE_DURATION "pause-duration"
+#define CMD_LINE_OPT_SCALE_FREQ_MIN "scale-freq-min"
+#define CMD_LINE_OPT_SCALE_FREQ_MAX "scale-freq-max"
 
 /* Parse the argument given in the command line of the application */
 static int
@@ -1825,6 +1855,10 @@ parse_args(int argc, char **argv)
 		{CMD_LINE_OPT_TELEMETRY, 0, 0, 0},
 		{CMD_LINE_OPT_INTERRUPT_ONLY, 0, 0, 0},
 		{CMD_LINE_OPT_PMD_MGMT, 1, 0, 0},
+		{CMD_LINE_OPT_MAX_EMPTY_POLLS, 1, 0, 0},
+		{CMD_LINE_OPT_PAUSE_DURATION, 1, 0, 0},
+		{CMD_LINE_OPT_SCALE_FREQ_MIN, 1, 0, 0},
+		{CMD_LINE_OPT_SCALE_FREQ_MAX, 1, 0, 0},
 		{NULL, 0, 0, 0}
 	};
 
@@ -1975,6 +2009,34 @@ parse_args(int argc, char **argv)
 				parse_ptype = 1;
 			}
 
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_MAX_EMPTY_POLLS,
+					sizeof(CMD_LINE_OPT_MAX_EMPTY_POLLS))) {
+				printf("Maximum empty polls configured\n");
+				max_empty_polls = parse_int(optarg);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_PAUSE_DURATION,
+					sizeof(CMD_LINE_OPT_PAUSE_DURATION))) {
+				printf("Pause duration configured\n");
+				pause_duration = parse_int(optarg);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_SCALE_FREQ_MIN,
+					sizeof(CMD_LINE_OPT_SCALE_FREQ_MIN))) {
+				printf("Scaling frequency minimum configured\n");
+				scale_freq_min = parse_int(optarg);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_SCALE_FREQ_MAX,
+					sizeof(CMD_LINE_OPT_SCALE_FREQ_MAX))) {
+				printf("Scaling frequency maximum configured\n");
+				scale_freq_max = parse_int(optarg);
+			}
+
 			break;
 
 		default:
@@ -2801,6 +2863,28 @@ main(int argc, char **argv)
 			}
 
 			if (app_mode == APP_MODE_PMD_MGMT && !baseline_enabled) {
+				/* Set power_pmd_mgmt configs passed by user */
+				rte_power_pmd_mgmt_set_emptypoll_max(max_empty_polls);
+				ret = rte_power_pmd_mgmt_set_pause_duration(pause_duration);
+				if (ret < 0)
+					rte_exit(EXIT_FAILURE,
+						"Error setting pause_duration: err=%d, lcore=%d\n",
+							ret, lcore_id);
+
+				ret = rte_power_pmd_mgmt_set_scaling_freq_min(lcore_id,
+						scale_freq_min);
+				if (ret < 0)
+					rte_exit(EXIT_FAILURE,
+						"Error setting scaling freq min: err=%d, lcore=%d\n",
+							ret, lcore_id);
+
+				ret = rte_power_pmd_mgmt_set_scaling_freq_max(lcore_id,
+						scale_freq_max);
+				if (ret < 0)
+					rte_exit(EXIT_FAILURE,
+						"Error setting scaling freq max: err=%d, lcore %d\n",
+							ret, lcore_id);
+
 				ret = rte_power_ethdev_pmgmt_queue_enable(
 						lcore_id, portid, queueid,
 						pmgmt_type);
-- 
2.31.1

Re: [PATCH v4 4/4] examples/l3fwd_power: add cli for configurable options

[David Hunt]

On 24/05/2022 14:14, Kevin Laatz wrote:
> Add CLI options to l3fwd_power to utilize the new power APIs introduced in
> this patchset. These CLI options allow the user to configure the
> heuritstics made available through the new API via the l3fwd_power
> application options.
>
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
>
> ---
> v3: move setters out of arg parsing
> v2: add doc update for l3fwd-power
> ---
>   .../sample_app_ug/l3_forward_power_man.rst    |  8 ++
>   examples/l3fwd-power/main.c                   | 86 ++++++++++++++++++-
>   2 files changed, 93 insertions(+), 1 deletion(-)
>

Tested-by: David Hunt <david.hunt@intel.com>

Re: [PATCH v4 0/4] Add APIs for configurable power options

[Burakov, Anatoly]

On 24-May-22 2:14 PM, Kevin Laatz wrote:
> The power library contains some variables which are currently set by
> defines, hard-coded values or set using sysfs values. In order to
> configure these, code changes and recompiles are required, making
> configuring these variables tedious.
> 
> This patchset introduces some new get/set APIs which allow users and
> applications to configure there settings to suit their use-cases.
> In addition, CLI options have been added to l3fwd_power to demonstrate
> how an application could use these APIs to expose the options to users
> without needing code changes to configure them.
> 
> ---
> v4:
> * fix return value when scaling_freq_max is not set
> * fix mismatching comments
> 
> v3:
> * move setters from arg parse function to init
> * consider 0 as 'not set' for scaling_freq_max
> * other minor fixes
> 
> v2:
> * add doc update for l3fwd-power
> * order version.map additions alphabetically
> 
> Kevin Laatz (4):
>    lib/power: add get and set API for emptypoll max
>    lib/power: add get and set API for pause duration
>    lib/power: add get and set API for scaling freq min and max with
>      pstate mode
>    examples/l3fwd_power: add cli for configurable options
> 
>   .../sample_app_ug/l3_forward_power_man.rst    |   8 +
>   examples/l3fwd-power/main.c                   |  86 ++++++++++-
>   lib/power/power_pstate_cpufreq.c              |  24 ++-
>   lib/power/rte_power_pmd_mgmt.c                | 123 +++++++++++++++-
>   lib/power/rte_power_pmd_mgmt.h                | 139 ++++++++++++++++++
>   lib/power/version.map                         |  10 ++
>   6 files changed, 379 insertions(+), 11 deletions(-)
> 

Hi Kevin,

We also have documentation for this library under 
`doc/guides/prog_guide/power_man.rst`, which documents methods exposed 
by the PMD power management. That document needs to be updated with the 
new additions :)

-- 
Thanks,
Anatoly

[PATCH v5 0/4] Add APIs for configurable power options

[Kevin Laatz]

The power library contains some variables which are currently set by
defines, hard-coded values or set using sysfs values. In order to
configure these, code changes and recompiles are required, making
configuring these variables tedious.

This patchset introduces some new get/set APIs which allow users and
applications to configure there settings to suit their use-cases.
In addition, CLI options have been added to l3fwd_power to demonstrate
how an application could use these APIs to expose the options to users
without needing code changes to configure them.

---
v5:
* add doc updates for new APIs

v4:
* fix return value when scaling_freq_max is not set
* fix mismatching comments

v3:
* move setters from arg parse function to init
* consider 0 as 'not set' for scaling_freq_max
* other minor fixes

v2:
* add doc update for l3fwd-power
* order version.map additions alphabetically

Kevin Laatz (4):
  lib/power: add get and set API for emptypoll max
  lib/power: add get and set API for pause duration
  lib/power: add get and set API for scaling freq min and max with
    pstate mode
  examples/l3fwd_power: add cli for configurable options

 doc/guides/prog_guide/power_man.rst           |  24 +++
 .../sample_app_ug/l3_forward_power_man.rst    |   8 +
 examples/l3fwd-power/main.c                   |  86 ++++++++++-
 lib/power/power_pstate_cpufreq.c              |  24 ++-
 lib/power/rte_power_pmd_mgmt.c                | 123 +++++++++++++++-
 lib/power/rte_power_pmd_mgmt.h                | 139 ++++++++++++++++++
 lib/power/version.map                         |  10 ++
 7 files changed, 403 insertions(+), 11 deletions(-)

-- 
2.31.1

[PATCH v5 1/4] lib/power: add get and set API for emptypoll max

[Kevin Laatz]

Add new get/set APIs to configure emptypoll max which is used to
determine when a queue can go into sleep state.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Ray Kinsella <mdr@ashroe.eu>
Tested-by: David Hunt <david.hunt@intel.com>

---
v5: add doc entry for new API
---
 doc/guides/prog_guide/power_man.rst |  6 ++++++
 lib/power/rte_power_pmd_mgmt.c      | 21 ++++++++++++++++++---
 lib/power/rte_power_pmd_mgmt.h      | 27 +++++++++++++++++++++++++++
 lib/power/version.map               |  4 ++++
 4 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/doc/guides/prog_guide/power_man.rst b/doc/guides/prog_guide/power_man.rst
index 4484a94dc8..5d345f6eb9 100644
--- a/doc/guides/prog_guide/power_man.rst
+++ b/doc/guides/prog_guide/power_man.rst
@@ -252,6 +252,12 @@ API Overview for Ethernet PMD Power Management
 
 * **Queue Disable**: Disable power scheme for certain queue/port/core.
 
+* **Get Emptypoll Max**: Get the configured number of empty polls to wait before
+  entering sleep state.
+
+* **Set Emptypoll Max**: Set the number of empty polls to wait before entering
+  sleep state.
+
 References
 ----------
 
diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index 39a2b4cd23..dfb7ca9187 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -11,7 +11,7 @@
 
 #include "rte_power_pmd_mgmt.h"
 
-#define EMPTYPOLL_MAX  512
+unsigned int emptypoll_max;
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -206,7 +206,7 @@ queue_can_sleep(struct pmd_core_cfg *cfg, struct queue_list_entry *qcfg)
 	qcfg->n_empty_polls++;
 
 	/* if we haven't reached threshold for empty polls, we can't sleep */
-	if (qcfg->n_empty_polls <= EMPTYPOLL_MAX)
+	if (qcfg->n_empty_polls <= emptypoll_max)
 		return false;
 
 	/*
@@ -290,7 +290,7 @@ clb_umwait(uint16_t port_id, uint16_t qidx, struct rte_mbuf **pkts __rte_unused,
 	/* this callback can't do more than one queue, omit multiqueue logic */
 	if (unlikely(nb_rx == 0)) {
 		queue_conf->n_empty_polls++;
-		if (unlikely(queue_conf->n_empty_polls > EMPTYPOLL_MAX)) {
+		if (unlikely(queue_conf->n_empty_polls > emptypoll_max)) {
 			struct rte_power_monitor_cond pmc;
 			int ret;
 
@@ -661,6 +661,18 @@ rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
 	return 0;
 }
 
+void
+rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max)
+{
+	emptypoll_max = max;
+}
+
+unsigned int
+rte_power_pmd_mgmt_get_emptypoll_max(void)
+{
+	return emptypoll_max;
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
 
@@ -669,4 +681,7 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 		struct pmd_core_cfg *cfg = &lcore_cfgs[i];
 		TAILQ_INIT(&cfg->head);
 	}
+
+	/* initialize config defaults */
+	emptypoll_max = 512;
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index 444e7b8a66..d5a94f8187 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -90,6 +90,33 @@ int
 rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
 		uint16_t port_id, uint16_t queue_id);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set a emptypoll_max to specified value. Used to specify the number of empty
+ * polls to wait before entering sleep state.
+ *
+ * @param max
+ *   The value to set emptypoll_max to.
+ */
+__rte_experimental
+void
+rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current value of emptypoll_max.
+ *
+ * @return
+ *   The current emptypoll_max value
+ */
+__rte_experimental
+unsigned int
+rte_power_pmd_mgmt_get_emptypoll_max(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 6ec6d5d96d..812843c3f3 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -38,4 +38,8 @@ EXPERIMENTAL {
 	# added in 21.02
 	rte_power_ethdev_pmgmt_queue_disable;
 	rte_power_ethdev_pmgmt_queue_enable;
+
+	# added in 22.07
+	rte_power_pmd_mgmt_get_emptypoll_max;
+	rte_power_pmd_mgmt_set_emptypoll_max;
 };
-- 
2.31.1

[PATCH v5 2/4] lib/power: add get and set API for pause duration

[Kevin Laatz]

Add new get/set API for configuring 'pause_duration' which used to adjust
the pause mode callback duration.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Ray Kinsella <mdr@ashroe.eu>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
Tested-by: David Hunt <david.hunt@intel.com>

---
v5: add doc entry for new API
v3: changed printf to RTE_LOG
---
 doc/guides/prog_guide/power_man.rst |  6 ++++++
 lib/power/rte_power_pmd_mgmt.c      | 25 +++++++++++++++++++++--
 lib/power/rte_power_pmd_mgmt.h      | 31 +++++++++++++++++++++++++++++
 lib/power/version.map               |  2 ++
 4 files changed, 62 insertions(+), 2 deletions(-)

diff --git a/doc/guides/prog_guide/power_man.rst b/doc/guides/prog_guide/power_man.rst
index 5d345f6eb9..f22513b324 100644
--- a/doc/guides/prog_guide/power_man.rst
+++ b/doc/guides/prog_guide/power_man.rst
@@ -258,6 +258,12 @@ API Overview for Ethernet PMD Power Management
 * **Set Emptypoll Max**: Set the number of empty polls to wait before entering
   sleep state.
 
+* **Get Pause Duration**: Get the configured duration (ms) to be used in the
+  Pause callback.
+
+* **Set Pause Duration**: Set the duration of the pause (ms) used in the Pause
+  mode callback.
+
 References
 ----------
 
diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index dfb7ca9187..1374cc213d 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -12,6 +12,7 @@
 #include "rte_power_pmd_mgmt.h"
 
 unsigned int emptypoll_max;
+unsigned int pause_duration;
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -315,6 +316,7 @@ clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,
 	struct queue_list_entry *queue_conf = arg;
 	struct pmd_core_cfg *lcore_conf;
 	const bool empty = nb_rx == 0;
+	uint32_t pause_duration = rte_power_pmd_mgmt_get_pause_duration();
 
 	lcore_conf = &lcore_cfgs[lcore];
 
@@ -334,11 +336,11 @@ clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,
 		if (global_data.intrinsics_support.power_pause) {
 			const uint64_t cur = rte_rdtsc();
 			const uint64_t wait_tsc =
-					cur + global_data.tsc_per_us;
+					cur + global_data.tsc_per_us * pause_duration;
 			rte_power_pause(wait_tsc);
 		} else {
 			uint64_t i;
-			for (i = 0; i < global_data.pause_per_us; i++)
+			for (i = 0; i < global_data.pause_per_us * pause_duration; i++)
 				rte_pause();
 		}
 	}
@@ -673,6 +675,24 @@ rte_power_pmd_mgmt_get_emptypoll_max(void)
 	return emptypoll_max;
 }
 
+int
+rte_power_pmd_mgmt_set_pause_duration(unsigned int duration)
+{
+	if (duration == 0) {
+		RTE_LOG(ERR, POWER, "Pause duration must be greater than 0, value unchanged");
+		return -EINVAL;
+	}
+	pause_duration = duration;
+
+	return 0;
+}
+
+unsigned int
+rte_power_pmd_mgmt_get_pause_duration(void)
+{
+	return pause_duration;
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
 
@@ -684,4 +704,5 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 
 	/* initialize config defaults */
 	emptypoll_max = 512;
+	pause_duration = 1;
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index d5a94f8187..18a9c3abb5 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -117,6 +117,37 @@ __rte_experimental
 unsigned int
 rte_power_pmd_mgmt_get_emptypoll_max(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the pause_duration. Used to adjust the pause mode callback duration.
+ *
+ * @note Duration must be greater than zero.
+ *
+ * @param duration
+ *   The value to set pause_duration to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_pause_duration(unsigned int duration);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current value of pause_duration.
+ *
+ * @return
+ *   The current pause_duration value.
+ */
+__rte_experimental
+unsigned int
+rte_power_pmd_mgmt_get_pause_duration(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 812843c3f3..4673b719f9 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -41,5 +41,7 @@ EXPERIMENTAL {
 
 	# added in 22.07
 	rte_power_pmd_mgmt_get_emptypoll_max;
+	rte_power_pmd_mgmt_get_pause_duration;
 	rte_power_pmd_mgmt_set_emptypoll_max;
+	rte_power_pmd_mgmt_set_pause_duration;
 };
-- 
2.31.1

Re: [PATCH v5 2/4] lib/power: add get and set API for pause duration

[Burakov, Anatoly]

On 31-May-22 10:59 AM, Kevin Laatz wrote:
> Add new get/set API for configuring 'pause_duration' which used to adjust
> the pause mode callback duration.
> 
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> Acked-by: Ray Kinsella <mdr@ashroe.eu>
> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
> Tested-by: David Hunt <david.hunt@intel.com>
> 
> ---
> v5: add doc entry for new API
> v3: changed printf to RTE_LOG
> ---
>   doc/guides/prog_guide/power_man.rst |  6 ++++++
>   lib/power/rte_power_pmd_mgmt.c      | 25 +++++++++++++++++++++--
>   lib/power/rte_power_pmd_mgmt.h      | 31 +++++++++++++++++++++++++++++
>   lib/power/version.map               |  2 ++
>   4 files changed, 62 insertions(+), 2 deletions(-)
> 
> diff --git a/doc/guides/prog_guide/power_man.rst b/doc/guides/prog_guide/power_man.rst
> index 5d345f6eb9..f22513b324 100644
> --- a/doc/guides/prog_guide/power_man.rst
> +++ b/doc/guides/prog_guide/power_man.rst
> @@ -258,6 +258,12 @@ API Overview for Ethernet PMD Power Management
>   * **Set Emptypoll Max**: Set the number of empty polls to wait before entering
>     sleep state.
>   
> +* **Get Pause Duration**: Get the configured duration (ms) to be used in the
> +  Pause callback.
> +
> +* **Set Pause Duration**: Set the duration of the pause (ms) used in the Pause
> +  mode callback.
> +

Is it not microseconds rather than milliseconds?

-- 
Thanks,
Anatoly

Re: [PATCH v5 2/4] lib/power: add get and set API for pause duration

[Kevin Laatz]

On 02/06/2022 15:01, Burakov, Anatoly wrote:
> On 31-May-22 10:59 AM, Kevin Laatz wrote:
>> Add new get/set API for configuring 'pause_duration' which used to 
>> adjust
>> the pause mode callback duration.
>>
>> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
>> Acked-by: Ray Kinsella <mdr@ashroe.eu>
>> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
>> Tested-by: David Hunt <david.hunt@intel.com>
>>
>> ---
>> v5: add doc entry for new API
>> v3: changed printf to RTE_LOG
>> ---
>>   doc/guides/prog_guide/power_man.rst |  6 ++++++
>>   lib/power/rte_power_pmd_mgmt.c      | 25 +++++++++++++++++++++--
>>   lib/power/rte_power_pmd_mgmt.h      | 31 +++++++++++++++++++++++++++++
>>   lib/power/version.map               |  2 ++
>>   4 files changed, 62 insertions(+), 2 deletions(-)
>>
>> diff --git a/doc/guides/prog_guide/power_man.rst 
>> b/doc/guides/prog_guide/power_man.rst
>> index 5d345f6eb9..f22513b324 100644
>> --- a/doc/guides/prog_guide/power_man.rst
>> +++ b/doc/guides/prog_guide/power_man.rst
>> @@ -258,6 +258,12 @@ API Overview for Ethernet PMD Power Management
>>   * **Set Emptypoll Max**: Set the number of empty polls to wait 
>> before entering
>>     sleep state.
>>   +* **Get Pause Duration**: Get the configured duration (ms) to be 
>> used in the
>> +  Pause callback.
>> +
>> +* **Set Pause Duration**: Set the duration of the pause (ms) used in 
>> the Pause
>> +  mode callback.
>> +
>
> Is it not microseconds rather than milliseconds?

Yes, you're right - fix coming. Thanks

/Kevin

[PATCH v5 3/4] lib/power: add get and set API for scaling freq min and max with pstate mode

[Kevin Laatz]

Add new get/set API to allow the user or application to set the minimum
and maximum frequencies to use when scaling.
Previously, the frequency range was determined by the HW capabilities of
the CPU. With this new API, the user or application can constrain this
if required.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Ray Kinsella <mdr@ashroe.eu>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
Tested-by: David Hunt <david.hunt@intel.com>

---
v5:
 * add doc entry for new API

v4:
 * fix mismatch between comments and code
 * fix return value when max freq is not set

v3:
 * updated doxygen comments
 * added checks to ensure min/max value is valid
 * consider 0 as 'not set' for scaling_freq_max
---
 doc/guides/prog_guide/power_man.rst | 12 +++++
 lib/power/power_pstate_cpufreq.c    | 24 +++++++--
 lib/power/rte_power_pmd_mgmt.c      | 77 +++++++++++++++++++++++++++
 lib/power/rte_power_pmd_mgmt.h      | 81 +++++++++++++++++++++++++++++
 lib/power/version.map               |  4 ++
 5 files changed, 193 insertions(+), 5 deletions(-)

diff --git a/doc/guides/prog_guide/power_man.rst b/doc/guides/prog_guide/power_man.rst
index f22513b324..a63a830213 100644
--- a/doc/guides/prog_guide/power_man.rst
+++ b/doc/guides/prog_guide/power_man.rst
@@ -264,6 +264,18 @@ API Overview for Ethernet PMD Power Management
 * **Set Pause Duration**: Set the duration of the pause (ms) used in the Pause
   mode callback.
 
+* **Get Scaling Min Freq**: Get the configured minimum frequency to be used in
+  Frequency Scaling mode.
+
+* **Set Scaling Min Freq**: Set the minimum frequency to be used in Frequency
+  Scaling mode.
+
+* **Get Scaling Max Freq**: Get the configured maximum frequency to be used in
+  Frequency Scaling mode.
+
+* **Set Scaling Max Freq**: Set the maximum frequency to be used in Frequency
+  Scaling mode.
+
 References
 ----------
 
diff --git a/lib/power/power_pstate_cpufreq.c b/lib/power/power_pstate_cpufreq.c
index f4c36179ec..78c9197695 100644
--- a/lib/power/power_pstate_cpufreq.c
+++ b/lib/power/power_pstate_cpufreq.c
@@ -12,6 +12,7 @@
 
 #include <rte_memcpy.h>
 
+#include "rte_power_pmd_mgmt.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
 
@@ -354,6 +355,7 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	FILE *f_min = NULL, *f_max = NULL;
 	int ret = -1;
 	uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
+	int config_min_freq, config_max_freq;
 	uint32_t i, num_freqs = 0;
 
 	/* open all files */
@@ -388,6 +390,18 @@ power_get_available_freqs(struct pstate_power_info *pi)
 		goto out;
 	}
 
+	/* check for config set by user or application to limit frequency range */
+	config_min_freq = rte_power_pmd_mgmt_get_scaling_freq_min(pi->lcore_id);
+	if (config_min_freq < 0)
+		goto out;
+	config_max_freq = rte_power_pmd_mgmt_get_scaling_freq_max(pi->lcore_id);
+	if (config_max_freq < 0)
+		goto out;
+
+	sys_min_freq = RTE_MAX(sys_min_freq, (uint32_t)config_min_freq);
+	if (config_max_freq > 0) /* Only use config_max_freq if a value has been set */
+		sys_max_freq = RTE_MIN(sys_max_freq, (uint32_t)config_max_freq);
+
 	if (sys_max_freq < sys_min_freq)
 		goto out;
 
@@ -411,8 +425,8 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	/* If turbo is available then there is one extra freq bucket
 	 * to store the sys max freq which value is base_max +1
 	 */
-	num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
-		pi->turbo_available;
+	num_freqs = (RTE_MIN(base_max_freq, sys_max_freq) - sys_min_freq) / BUS_FREQ
+			+ 1 + pi->turbo_available;
 	if (num_freqs >= RTE_MAX_LCORE_FREQS) {
 		RTE_LOG(ERR, POWER, "Too many available frequencies: %d\n",
 				num_freqs);
@@ -427,10 +441,10 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	 */
 	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 		if ((i == 0) && pi->turbo_available)
-			pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
+			pi->freqs[pi->nb_freqs++] = RTE_MIN(base_max_freq, sys_max_freq) + 1;
 		else
-			pi->freqs[pi->nb_freqs++] =
-			base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
+			pi->freqs[pi->nb_freqs++] = RTE_MIN(base_max_freq, sys_max_freq) -
+					(i - pi->turbo_available) * BUS_FREQ;
 	}
 
 	ret = 0;
diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index 1374cc213d..a66e08e574 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -10,9 +10,12 @@
 #include <rte_power_intrinsics.h>
 
 #include "rte_power_pmd_mgmt.h"
+#include "power_common.h"
 
 unsigned int emptypoll_max;
 unsigned int pause_duration;
+unsigned int scale_freq_min[RTE_MAX_LCORE];
+unsigned int scale_freq_max[RTE_MAX_LCORE];
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -693,8 +696,77 @@ rte_power_pmd_mgmt_get_pause_duration(void)
 	return pause_duration;
 }
 
+int
+rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	if (min > scale_freq_max[lcore]) {
+		RTE_LOG(ERR, POWER, "Invalid min frequency: Cannot be greater than max frequency");
+		return -EINVAL;
+	}
+	scale_freq_min[lcore] = min;
+
+	return 0;
+}
+
+int
+rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	/* Zero means 'not set'. Use UINT32_MAX to enable RTE_MIN/MAX macro use when scaling. */
+	if (max == 0)
+		max = UINT32_MAX;
+	if (max < scale_freq_min[lcore]) {
+		RTE_LOG(ERR, POWER, "Invalid max frequency: Cannot be less than min frequency");
+		return -EINVAL;
+	}
+
+	scale_freq_max[lcore] = max;
+
+	return 0;
+}
+
+int
+rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	if (scale_freq_max[lcore] == 0)
+		RTE_LOG(DEBUG, POWER, "Scaling freq min config not set. Using sysfs min freq.\n");
+
+	return scale_freq_min[lcore];
+}
+
+int
+rte_power_pmd_mgmt_get_scaling_freq_max(unsigned int lcore)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	if (scale_freq_max[lcore] == UINT32_MAX) {
+		RTE_LOG(DEBUG, POWER, "Scaling freq max config not set. Using sysfs max freq.\n");
+		return 0;
+	}
+
+	return scale_freq_max[lcore];
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
+	int j;
 
 	/* initialize all tailqs */
 	for (i = 0; i < RTE_DIM(lcore_cfgs); i++) {
@@ -705,4 +777,9 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	/* initialize config defaults */
 	emptypoll_max = 512;
 	pause_duration = 1;
+	/* scaling defaults out of range to ensure not used unless set by user or app */
+	for (j = 0; j < RTE_MAX_LCORE; j++) {
+		scale_freq_min[j] = 0;
+		scale_freq_max[j] = UINT32_MAX;
+	}
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index 18a9c3abb5..789fbe0e4d 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -148,6 +148,87 @@ __rte_experimental
 unsigned int
 rte_power_pmd_mgmt_get_pause_duration(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the min frequency to be used for frequency scaling or zero to use defaults.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to set the min frequency for.
+ * @param min
+ *   The value, in KiloHertz, to set the minimum frequency to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the max frequency to be used for frequency scaling or zero to use defaults.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to set the max frequency for.
+ * @param max
+ *   The value, in KiloHertz, to set the maximum frequency to.
+ *   If 'max' is 0, it is considered 'not set'.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current configured min frequency used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to get the min frequency for.
+ * @return
+ *   0 if no value has been configured via the 'set' API.
+ *   >0 if a minimum frequency has been configured. Value is the minimum frequency
+ *   , in KiloHertz, used for frequency scaling.
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current configured max frequency used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to get the max frequency for.
+ * @return
+ *   0 if no value has been configured via the 'set' API.
+ *   >0 if a maximum frequency has been configured. Value is the maximum frequency
+ *   , in KiloHertz, used for frequency scaling.
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_get_scaling_freq_max(unsigned int lcore);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 4673b719f9..a687754f4a 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -42,6 +42,10 @@ EXPERIMENTAL {
 	# added in 22.07
 	rte_power_pmd_mgmt_get_emptypoll_max;
 	rte_power_pmd_mgmt_get_pause_duration;
+	rte_power_pmd_mgmt_get_scaling_freq_max;
+	rte_power_pmd_mgmt_get_scaling_freq_min;
 	rte_power_pmd_mgmt_set_emptypoll_max;
 	rte_power_pmd_mgmt_set_pause_duration;
+	rte_power_pmd_mgmt_set_scaling_freq_max;
+	rte_power_pmd_mgmt_set_scaling_freq_min;
 };
-- 
2.31.1

[PATCH v5 4/4] examples/l3fwd_power: add cli for configurable options

[Kevin Laatz]

Add CLI options to l3fwd_power to utilize the new power APIs introduced in
this patchset. These CLI options allow the user to configure the
heuritstics made available through the new API via the l3fwd_power
application options.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

---
v3: move setters out of arg parsing
v2: add doc update for l3fwd-power
---
 .../sample_app_ug/l3_forward_power_man.rst    |  8 ++
 examples/l3fwd-power/main.c                   | 86 ++++++++++++++++++-
 2 files changed, 93 insertions(+), 1 deletion(-)

diff --git a/doc/guides/sample_app_ug/l3_forward_power_man.rst b/doc/guides/sample_app_ug/l3_forward_power_man.rst
index 2e350c45f1..8f6d906200 100644
--- a/doc/guides/sample_app_ug/l3_forward_power_man.rst
+++ b/doc/guides/sample_app_ug/l3_forward_power_man.rst
@@ -109,6 +109,14 @@ where,
 
 *   --pmd-mgmt: PMD power management mode.
 
+*   --max-empty-polls : Number of empty polls to wait before entering sleep state. Applies to --pmd-mgmt mode only.
+
+*   --pause-duration: Set the duration of the pause callback (microseconds). Applies to --pmd-mgmt mode only.
+
+*   --scale-freq-min: Set minimum frequency for scaling. Applies to --pmd-mgmt mode only.
+
+*   --scale-freq-max: Set maximum frequency for scaling. Applies to --pmd-mgmt mode only.
+
 See :doc:`l3_forward` for details.
 The L3fwd-power example reuses the L3fwd command line options.
 
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 20e5b59af9..887c6eae3f 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -265,6 +265,10 @@ static struct rte_eth_conf port_conf = {
 };
 
 static uint32_t max_pkt_len;
+static uint32_t max_empty_polls = 512;
+static uint32_t pause_duration = 1;
+static uint32_t scale_freq_min;
+static uint32_t scale_freq_max;
 
 static struct rte_mempool * pktmbuf_pool[NB_SOCKETS];
 
@@ -1626,10 +1630,32 @@ print_usage(const char *prgname)
 		" empty polls, full polls, and core busyness to telemetry\n"
 		" --interrupt-only: enable interrupt-only mode\n"
 		" --pmd-mgmt MODE: enable PMD power management mode. "
-		"Currently supported modes: baseline, monitor, pause, scale\n",
+		"Currently supported modes: baseline, monitor, pause, scale\n"
+		"  --max-empty-polls MAX_EMPTY_POLLS: number of empty polls to"
+		" wait before entering sleep state\n"
+		"  --pause-duration DURATION: set the duration, in microseconds,"
+		" of the pause callback\n"
+		"  --scale-freq-min FREQ_MIN: set minimum frequency for scaling mode for"
+		" all application lcores (FREQ_MIN must be in kHz, in increments of 100MHz)\n"
+		"  --scale-freq-max FREQ_MAX: set maximum frequency for scaling mode for"
+		" all application lcores (FREQ_MAX must be in kHz, in increments of 100MHz)\n",
 		prgname);
 }
 
+static int
+parse_int(const char *opt)
+{
+	char *end = NULL;
+	unsigned long val;
+
+	/* parse integer string */
+	val = strtoul(opt, &end, 10);
+	if ((opt[0] == '\0') || (end == NULL) || (*end != '\0'))
+		return -1;
+
+	return val;
+}
+
 static int parse_max_pkt_len(const char *pktlen)
 {
 	char *end = NULL;
@@ -1803,6 +1829,10 @@ parse_ep_config(const char *q_arg)
 #define CMD_LINE_OPT_TELEMETRY "telemetry"
 #define CMD_LINE_OPT_PMD_MGMT "pmd-mgmt"
 #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len"
+#define CMD_LINE_OPT_MAX_EMPTY_POLLS "max-empty-polls"
+#define CMD_LINE_OPT_PAUSE_DURATION "pause-duration"
+#define CMD_LINE_OPT_SCALE_FREQ_MIN "scale-freq-min"
+#define CMD_LINE_OPT_SCALE_FREQ_MAX "scale-freq-max"
 
 /* Parse the argument given in the command line of the application */
 static int
@@ -1825,6 +1855,10 @@ parse_args(int argc, char **argv)
 		{CMD_LINE_OPT_TELEMETRY, 0, 0, 0},
 		{CMD_LINE_OPT_INTERRUPT_ONLY, 0, 0, 0},
 		{CMD_LINE_OPT_PMD_MGMT, 1, 0, 0},
+		{CMD_LINE_OPT_MAX_EMPTY_POLLS, 1, 0, 0},
+		{CMD_LINE_OPT_PAUSE_DURATION, 1, 0, 0},
+		{CMD_LINE_OPT_SCALE_FREQ_MIN, 1, 0, 0},
+		{CMD_LINE_OPT_SCALE_FREQ_MAX, 1, 0, 0},
 		{NULL, 0, 0, 0}
 	};
 
@@ -1975,6 +2009,34 @@ parse_args(int argc, char **argv)
 				parse_ptype = 1;
 			}
 
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_MAX_EMPTY_POLLS,
+					sizeof(CMD_LINE_OPT_MAX_EMPTY_POLLS))) {
+				printf("Maximum empty polls configured\n");
+				max_empty_polls = parse_int(optarg);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_PAUSE_DURATION,
+					sizeof(CMD_LINE_OPT_PAUSE_DURATION))) {
+				printf("Pause duration configured\n");
+				pause_duration = parse_int(optarg);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_SCALE_FREQ_MIN,
+					sizeof(CMD_LINE_OPT_SCALE_FREQ_MIN))) {
+				printf("Scaling frequency minimum configured\n");
+				scale_freq_min = parse_int(optarg);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_SCALE_FREQ_MAX,
+					sizeof(CMD_LINE_OPT_SCALE_FREQ_MAX))) {
+				printf("Scaling frequency maximum configured\n");
+				scale_freq_max = parse_int(optarg);
+			}
+
 			break;
 
 		default:
@@ -2801,6 +2863,28 @@ main(int argc, char **argv)
 			}
 
 			if (app_mode == APP_MODE_PMD_MGMT && !baseline_enabled) {
+				/* Set power_pmd_mgmt configs passed by user */
+				rte_power_pmd_mgmt_set_emptypoll_max(max_empty_polls);
+				ret = rte_power_pmd_mgmt_set_pause_duration(pause_duration);
+				if (ret < 0)
+					rte_exit(EXIT_FAILURE,
+						"Error setting pause_duration: err=%d, lcore=%d\n",
+							ret, lcore_id);
+
+				ret = rte_power_pmd_mgmt_set_scaling_freq_min(lcore_id,
+						scale_freq_min);
+				if (ret < 0)
+					rte_exit(EXIT_FAILURE,
+						"Error setting scaling freq min: err=%d, lcore=%d\n",
+							ret, lcore_id);
+
+				ret = rte_power_pmd_mgmt_set_scaling_freq_max(lcore_id,
+						scale_freq_max);
+				if (ret < 0)
+					rte_exit(EXIT_FAILURE,
+						"Error setting scaling freq max: err=%d, lcore %d\n",
+							ret, lcore_id);
+
 				ret = rte_power_ethdev_pmgmt_queue_enable(
 						lcore_id, portid, queueid,
 						pmgmt_type);
-- 
2.31.1

[PATCH v6 0/4] Add APIs for configurable power options

[Kevin Laatz]

The power library contains some variables which are currently set by
defines, hard-coded values or set using sysfs values. In order to
configure these, code changes and recompiles are required, making
configuring these variables tedious.

This patchset introduces some new get/set APIs which allow users and
applications to configure there settings to suit their use-cases.
In addition, CLI options have been added to l3fwd_power to demonstrate
how an application could use these APIs to expose the options to users
without needing code changes to configure them.

---
v6:
* fix units in doc API descriptions

v5:
* add doc updates for new APIs

v4:
* fix return value when scaling_freq_max is not set
* fix mismatching comments

v3:
* move setters from arg parse function to init
* consider 0 as 'not set' for scaling_freq_max
* other minor fixes

v2:
* add doc update for l3fwd-power
* order version.map additions alphabetically

Kevin Laatz (4):
  lib/power: add get and set API for emptypoll max
  lib/power: add get and set API for pause duration
  lib/power: add get and set API for scaling freq min and max with
    pstate mode
  examples/l3fwd_power: add cli for configurable options

 doc/guides/prog_guide/power_man.rst           |  24 +++
 .../sample_app_ug/l3_forward_power_man.rst    |   8 +
 examples/l3fwd-power/main.c                   |  86 ++++++++++-
 lib/power/power_pstate_cpufreq.c              |  24 ++-
 lib/power/rte_power_pmd_mgmt.c                | 123 +++++++++++++++-
 lib/power/rte_power_pmd_mgmt.h                | 139 ++++++++++++++++++
 lib/power/version.map                         |  10 ++
 7 files changed, 403 insertions(+), 11 deletions(-)

-- 
2.31.1

[PATCH v6 1/4] lib/power: add get and set API for emptypoll max

[Kevin Laatz]

Add new get/set APIs to configure emptypoll max which is used to
determine when a queue can go into sleep state.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Ray Kinsella <mdr@ashroe.eu>
Tested-by: David Hunt <david.hunt@intel.com>

---
v5: add doc entry for new API
---
 doc/guides/prog_guide/power_man.rst |  6 ++++++
 lib/power/rte_power_pmd_mgmt.c      | 21 ++++++++++++++++++---
 lib/power/rte_power_pmd_mgmt.h      | 27 +++++++++++++++++++++++++++
 lib/power/version.map               |  4 ++++
 4 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/doc/guides/prog_guide/power_man.rst b/doc/guides/prog_guide/power_man.rst
index 4484a94dc8..5d345f6eb9 100644
--- a/doc/guides/prog_guide/power_man.rst
+++ b/doc/guides/prog_guide/power_man.rst
@@ -252,6 +252,12 @@ API Overview for Ethernet PMD Power Management
 
 * **Queue Disable**: Disable power scheme for certain queue/port/core.
 
+* **Get Emptypoll Max**: Get the configured number of empty polls to wait before
+  entering sleep state.
+
+* **Set Emptypoll Max**: Set the number of empty polls to wait before entering
+  sleep state.
+
 References
 ----------
 
diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index 39a2b4cd23..dfb7ca9187 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -11,7 +11,7 @@
 
 #include "rte_power_pmd_mgmt.h"
 
-#define EMPTYPOLL_MAX  512
+unsigned int emptypoll_max;
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -206,7 +206,7 @@ queue_can_sleep(struct pmd_core_cfg *cfg, struct queue_list_entry *qcfg)
 	qcfg->n_empty_polls++;
 
 	/* if we haven't reached threshold for empty polls, we can't sleep */
-	if (qcfg->n_empty_polls <= EMPTYPOLL_MAX)
+	if (qcfg->n_empty_polls <= emptypoll_max)
 		return false;
 
 	/*
@@ -290,7 +290,7 @@ clb_umwait(uint16_t port_id, uint16_t qidx, struct rte_mbuf **pkts __rte_unused,
 	/* this callback can't do more than one queue, omit multiqueue logic */
 	if (unlikely(nb_rx == 0)) {
 		queue_conf->n_empty_polls++;
-		if (unlikely(queue_conf->n_empty_polls > EMPTYPOLL_MAX)) {
+		if (unlikely(queue_conf->n_empty_polls > emptypoll_max)) {
 			struct rte_power_monitor_cond pmc;
 			int ret;
 
@@ -661,6 +661,18 @@ rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
 	return 0;
 }
 
+void
+rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max)
+{
+	emptypoll_max = max;
+}
+
+unsigned int
+rte_power_pmd_mgmt_get_emptypoll_max(void)
+{
+	return emptypoll_max;
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
 
@@ -669,4 +681,7 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 		struct pmd_core_cfg *cfg = &lcore_cfgs[i];
 		TAILQ_INIT(&cfg->head);
 	}
+
+	/* initialize config defaults */
+	emptypoll_max = 512;
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index 444e7b8a66..d5a94f8187 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -90,6 +90,33 @@ int
 rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,
 		uint16_t port_id, uint16_t queue_id);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set a emptypoll_max to specified value. Used to specify the number of empty
+ * polls to wait before entering sleep state.
+ *
+ * @param max
+ *   The value to set emptypoll_max to.
+ */
+__rte_experimental
+void
+rte_power_pmd_mgmt_set_emptypoll_max(unsigned int max);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current value of emptypoll_max.
+ *
+ * @return
+ *   The current emptypoll_max value
+ */
+__rte_experimental
+unsigned int
+rte_power_pmd_mgmt_get_emptypoll_max(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 6ec6d5d96d..812843c3f3 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -38,4 +38,8 @@ EXPERIMENTAL {
 	# added in 21.02
 	rte_power_ethdev_pmgmt_queue_disable;
 	rte_power_ethdev_pmgmt_queue_enable;
+
+	# added in 22.07
+	rte_power_pmd_mgmt_get_emptypoll_max;
+	rte_power_pmd_mgmt_set_emptypoll_max;
 };
-- 
2.31.1

[PATCH v6 2/4] lib/power: add get and set API for pause duration

[Kevin Laatz]

Add new get/set API for configuring 'pause_duration' which used to adjust
the pause mode callback duration.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Ray Kinsella <mdr@ashroe.eu>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
Tested-by: David Hunt <david.hunt@intel.com>

---
v6: fix unit of pause duration in doc
v5: add doc entry for new API
v3: changed printf to RTE_LOG
---
 doc/guides/prog_guide/power_man.rst |  6 ++++++
 lib/power/rte_power_pmd_mgmt.c      | 25 +++++++++++++++++++++--
 lib/power/rte_power_pmd_mgmt.h      | 31 +++++++++++++++++++++++++++++
 lib/power/version.map               |  2 ++
 4 files changed, 62 insertions(+), 2 deletions(-)

diff --git a/doc/guides/prog_guide/power_man.rst b/doc/guides/prog_guide/power_man.rst
index 5d345f6eb9..266c7544e5 100644
--- a/doc/guides/prog_guide/power_man.rst
+++ b/doc/guides/prog_guide/power_man.rst
@@ -258,6 +258,12 @@ API Overview for Ethernet PMD Power Management
 * **Set Emptypoll Max**: Set the number of empty polls to wait before entering
   sleep state.
 
+* **Get Pause Duration**: Get the configured duration (microseconds) to be used
+  in the Pause callback.
+
+* **Set Pause Duration**: Set the duration of the pause (microseconds) used in
+  the Pause mode callback.
+
 References
 ----------
 
diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index dfb7ca9187..1374cc213d 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -12,6 +12,7 @@
 #include "rte_power_pmd_mgmt.h"
 
 unsigned int emptypoll_max;
+unsigned int pause_duration;
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -315,6 +316,7 @@ clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,
 	struct queue_list_entry *queue_conf = arg;
 	struct pmd_core_cfg *lcore_conf;
 	const bool empty = nb_rx == 0;
+	uint32_t pause_duration = rte_power_pmd_mgmt_get_pause_duration();
 
 	lcore_conf = &lcore_cfgs[lcore];
 
@@ -334,11 +336,11 @@ clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,
 		if (global_data.intrinsics_support.power_pause) {
 			const uint64_t cur = rte_rdtsc();
 			const uint64_t wait_tsc =
-					cur + global_data.tsc_per_us;
+					cur + global_data.tsc_per_us * pause_duration;
 			rte_power_pause(wait_tsc);
 		} else {
 			uint64_t i;
-			for (i = 0; i < global_data.pause_per_us; i++)
+			for (i = 0; i < global_data.pause_per_us * pause_duration; i++)
 				rte_pause();
 		}
 	}
@@ -673,6 +675,24 @@ rte_power_pmd_mgmt_get_emptypoll_max(void)
 	return emptypoll_max;
 }
 
+int
+rte_power_pmd_mgmt_set_pause_duration(unsigned int duration)
+{
+	if (duration == 0) {
+		RTE_LOG(ERR, POWER, "Pause duration must be greater than 0, value unchanged");
+		return -EINVAL;
+	}
+	pause_duration = duration;
+
+	return 0;
+}
+
+unsigned int
+rte_power_pmd_mgmt_get_pause_duration(void)
+{
+	return pause_duration;
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
 
@@ -684,4 +704,5 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 
 	/* initialize config defaults */
 	emptypoll_max = 512;
+	pause_duration = 1;
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index d5a94f8187..18a9c3abb5 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -117,6 +117,37 @@ __rte_experimental
 unsigned int
 rte_power_pmd_mgmt_get_emptypoll_max(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the pause_duration. Used to adjust the pause mode callback duration.
+ *
+ * @note Duration must be greater than zero.
+ *
+ * @param duration
+ *   The value to set pause_duration to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_pause_duration(unsigned int duration);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current value of pause_duration.
+ *
+ * @return
+ *   The current pause_duration value.
+ */
+__rte_experimental
+unsigned int
+rte_power_pmd_mgmt_get_pause_duration(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 812843c3f3..4673b719f9 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -41,5 +41,7 @@ EXPERIMENTAL {
 
 	# added in 22.07
 	rte_power_pmd_mgmt_get_emptypoll_max;
+	rte_power_pmd_mgmt_get_pause_duration;
 	rte_power_pmd_mgmt_set_emptypoll_max;
+	rte_power_pmd_mgmt_set_pause_duration;
 };
-- 
2.31.1

[PATCH v6 3/4] lib/power: add get and set API for scaling freq min and max with pstate mode

[Kevin Laatz]

Add new get/set API to allow the user or application to set the minimum
and maximum frequencies to use when scaling.
Previously, the frequency range was determined by the HW capabilities of
the CPU. With this new API, the user or application can constrain this
if required.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Ray Kinsella <mdr@ashroe.eu>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
Tested-by: David Hunt <david.hunt@intel.com>

---
v6:
 * add frequency unit used by APIs to docs

v5:
 * add doc entry for new API

v4:
 * fix mismatch between comments and code
 * fix return value when max freq is not set

v3:
 * updated doxygen comments
 * added checks to ensure min/max value is valid
 * consider 0 as 'not set' for scaling_freq_max
---
 doc/guides/prog_guide/power_man.rst | 12 +++++
 lib/power/power_pstate_cpufreq.c    | 24 +++++++--
 lib/power/rte_power_pmd_mgmt.c      | 77 +++++++++++++++++++++++++++
 lib/power/rte_power_pmd_mgmt.h      | 81 +++++++++++++++++++++++++++++
 lib/power/version.map               |  4 ++
 5 files changed, 193 insertions(+), 5 deletions(-)

diff --git a/doc/guides/prog_guide/power_man.rst b/doc/guides/prog_guide/power_man.rst
index 266c7544e5..98cfd3c1f3 100644
--- a/doc/guides/prog_guide/power_man.rst
+++ b/doc/guides/prog_guide/power_man.rst
@@ -264,6 +264,18 @@ API Overview for Ethernet PMD Power Management
 * **Set Pause Duration**: Set the duration of the pause (microseconds) used in
   the Pause mode callback.
 
+* **Get Scaling Min Freq**: Get the configured minimum frequency (kHz) to be used
+  in Frequency Scaling mode.
+
+* **Set Scaling Min Freq**: Set the minimum frequency (kHz) to be used in Frequency
+  Scaling mode.
+
+* **Get Scaling Max Freq**: Get the configured maximum frequency (kHz) to be used
+  in Frequency Scaling mode.
+
+* **Set Scaling Max Freq**: Set the maximum frequency (kHz) to be used in Frequency
+  Scaling mode.
+
 References
 ----------
 
diff --git a/lib/power/power_pstate_cpufreq.c b/lib/power/power_pstate_cpufreq.c
index f4c36179ec..78c9197695 100644
--- a/lib/power/power_pstate_cpufreq.c
+++ b/lib/power/power_pstate_cpufreq.c
@@ -12,6 +12,7 @@
 
 #include <rte_memcpy.h>
 
+#include "rte_power_pmd_mgmt.h"
 #include "power_pstate_cpufreq.h"
 #include "power_common.h"
 
@@ -354,6 +355,7 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	FILE *f_min = NULL, *f_max = NULL;
 	int ret = -1;
 	uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
+	int config_min_freq, config_max_freq;
 	uint32_t i, num_freqs = 0;
 
 	/* open all files */
@@ -388,6 +390,18 @@ power_get_available_freqs(struct pstate_power_info *pi)
 		goto out;
 	}
 
+	/* check for config set by user or application to limit frequency range */
+	config_min_freq = rte_power_pmd_mgmt_get_scaling_freq_min(pi->lcore_id);
+	if (config_min_freq < 0)
+		goto out;
+	config_max_freq = rte_power_pmd_mgmt_get_scaling_freq_max(pi->lcore_id);
+	if (config_max_freq < 0)
+		goto out;
+
+	sys_min_freq = RTE_MAX(sys_min_freq, (uint32_t)config_min_freq);
+	if (config_max_freq > 0) /* Only use config_max_freq if a value has been set */
+		sys_max_freq = RTE_MIN(sys_max_freq, (uint32_t)config_max_freq);
+
 	if (sys_max_freq < sys_min_freq)
 		goto out;
 
@@ -411,8 +425,8 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	/* If turbo is available then there is one extra freq bucket
 	 * to store the sys max freq which value is base_max +1
 	 */
-	num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
-		pi->turbo_available;
+	num_freqs = (RTE_MIN(base_max_freq, sys_max_freq) - sys_min_freq) / BUS_FREQ
+			+ 1 + pi->turbo_available;
 	if (num_freqs >= RTE_MAX_LCORE_FREQS) {
 		RTE_LOG(ERR, POWER, "Too many available frequencies: %d\n",
 				num_freqs);
@@ -427,10 +441,10 @@ power_get_available_freqs(struct pstate_power_info *pi)
 	 */
 	for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 		if ((i == 0) && pi->turbo_available)
-			pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
+			pi->freqs[pi->nb_freqs++] = RTE_MIN(base_max_freq, sys_max_freq) + 1;
 		else
-			pi->freqs[pi->nb_freqs++] =
-			base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
+			pi->freqs[pi->nb_freqs++] = RTE_MIN(base_max_freq, sys_max_freq) -
+					(i - pi->turbo_available) * BUS_FREQ;
 	}
 
 	ret = 0;
diff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c
index 1374cc213d..a66e08e574 100644
--- a/lib/power/rte_power_pmd_mgmt.c
+++ b/lib/power/rte_power_pmd_mgmt.c
@@ -10,9 +10,12 @@
 #include <rte_power_intrinsics.h>
 
 #include "rte_power_pmd_mgmt.h"
+#include "power_common.h"
 
 unsigned int emptypoll_max;
 unsigned int pause_duration;
+unsigned int scale_freq_min[RTE_MAX_LCORE];
+unsigned int scale_freq_max[RTE_MAX_LCORE];
 
 /* store some internal state */
 static struct pmd_conf_data {
@@ -693,8 +696,77 @@ rte_power_pmd_mgmt_get_pause_duration(void)
 	return pause_duration;
 }
 
+int
+rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	if (min > scale_freq_max[lcore]) {
+		RTE_LOG(ERR, POWER, "Invalid min frequency: Cannot be greater than max frequency");
+		return -EINVAL;
+	}
+	scale_freq_min[lcore] = min;
+
+	return 0;
+}
+
+int
+rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	/* Zero means 'not set'. Use UINT32_MAX to enable RTE_MIN/MAX macro use when scaling. */
+	if (max == 0)
+		max = UINT32_MAX;
+	if (max < scale_freq_min[lcore]) {
+		RTE_LOG(ERR, POWER, "Invalid max frequency: Cannot be less than min frequency");
+		return -EINVAL;
+	}
+
+	scale_freq_max[lcore] = max;
+
+	return 0;
+}
+
+int
+rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	if (scale_freq_max[lcore] == 0)
+		RTE_LOG(DEBUG, POWER, "Scaling freq min config not set. Using sysfs min freq.\n");
+
+	return scale_freq_min[lcore];
+}
+
+int
+rte_power_pmd_mgmt_get_scaling_freq_max(unsigned int lcore)
+{
+	if (lcore >= RTE_MAX_LCORE) {
+		RTE_LOG(ERR, POWER, "Invalid lcore ID: %u\n", lcore);
+		return -EINVAL;
+	}
+
+	if (scale_freq_max[lcore] == UINT32_MAX) {
+		RTE_LOG(DEBUG, POWER, "Scaling freq max config not set. Using sysfs max freq.\n");
+		return 0;
+	}
+
+	return scale_freq_max[lcore];
+}
+
 RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	size_t i;
+	int j;
 
 	/* initialize all tailqs */
 	for (i = 0; i < RTE_DIM(lcore_cfgs); i++) {
@@ -705,4 +777,9 @@ RTE_INIT(rte_power_ethdev_pmgmt_init) {
 	/* initialize config defaults */
 	emptypoll_max = 512;
 	pause_duration = 1;
+	/* scaling defaults out of range to ensure not used unless set by user or app */
+	for (j = 0; j < RTE_MAX_LCORE; j++) {
+		scale_freq_min[j] = 0;
+		scale_freq_max[j] = UINT32_MAX;
+	}
 }
diff --git a/lib/power/rte_power_pmd_mgmt.h b/lib/power/rte_power_pmd_mgmt.h
index 18a9c3abb5..789fbe0e4d 100644
--- a/lib/power/rte_power_pmd_mgmt.h
+++ b/lib/power/rte_power_pmd_mgmt.h
@@ -148,6 +148,87 @@ __rte_experimental
 unsigned int
 rte_power_pmd_mgmt_get_pause_duration(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the min frequency to be used for frequency scaling or zero to use defaults.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to set the min frequency for.
+ * @param min
+ *   The value, in KiloHertz, to set the minimum frequency to.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_scaling_freq_min(unsigned int lcore, unsigned int min);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Set the max frequency to be used for frequency scaling or zero to use defaults.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to set the max frequency for.
+ * @param max
+ *   The value, in KiloHertz, to set the maximum frequency to.
+ *   If 'max' is 0, it is considered 'not set'.
+ * @return
+ *   0 on success
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_set_scaling_freq_max(unsigned int lcore, unsigned int max);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current configured min frequency used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to get the min frequency for.
+ * @return
+ *   0 if no value has been configured via the 'set' API.
+ *   >0 if a minimum frequency has been configured. Value is the minimum frequency
+ *   , in KiloHertz, used for frequency scaling.
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_get_scaling_freq_min(unsigned int lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice.
+ *
+ * Get the current configured max frequency used for frequency scaling.
+ *
+ * @note Supported by: Pstate mode.
+ *
+ * @param lcore
+ *   The ID of the lcore to get the max frequency for.
+ * @return
+ *   0 if no value has been configured via the 'set' API.
+ *   >0 if a maximum frequency has been configured. Value is the maximum frequency
+ *   , in KiloHertz, used for frequency scaling.
+ *   <0 on error
+ */
+__rte_experimental
+int
+rte_power_pmd_mgmt_get_scaling_freq_max(unsigned int lcore);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/power/version.map b/lib/power/version.map
index 4673b719f9..a687754f4a 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -42,6 +42,10 @@ EXPERIMENTAL {
 	# added in 22.07
 	rte_power_pmd_mgmt_get_emptypoll_max;
 	rte_power_pmd_mgmt_get_pause_duration;
+	rte_power_pmd_mgmt_get_scaling_freq_max;
+	rte_power_pmd_mgmt_get_scaling_freq_min;
 	rte_power_pmd_mgmt_set_emptypoll_max;
 	rte_power_pmd_mgmt_set_pause_duration;
+	rte_power_pmd_mgmt_set_scaling_freq_max;
+	rte_power_pmd_mgmt_set_scaling_freq_min;
 };
-- 
2.31.1

[PATCH v6 4/4] examples/l3fwd_power: add cli for configurable options

[Kevin Laatz]

Add CLI options to l3fwd_power to utilize the new power APIs introduced in
this patchset. These CLI options allow the user to configure the
heuritstics made available through the new API via the l3fwd_power
application options.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

---
v3: move setters out of arg parsing
v2: add doc update for l3fwd-power
---
 .../sample_app_ug/l3_forward_power_man.rst    |  8 ++
 examples/l3fwd-power/main.c                   | 86 ++++++++++++++++++-
 2 files changed, 93 insertions(+), 1 deletion(-)

diff --git a/doc/guides/sample_app_ug/l3_forward_power_man.rst b/doc/guides/sample_app_ug/l3_forward_power_man.rst
index 2e350c45f1..8f6d906200 100644
--- a/doc/guides/sample_app_ug/l3_forward_power_man.rst
+++ b/doc/guides/sample_app_ug/l3_forward_power_man.rst
@@ -109,6 +109,14 @@ where,
 
 *   --pmd-mgmt: PMD power management mode.
 
+*   --max-empty-polls : Number of empty polls to wait before entering sleep state. Applies to --pmd-mgmt mode only.
+
+*   --pause-duration: Set the duration of the pause callback (microseconds). Applies to --pmd-mgmt mode only.
+
+*   --scale-freq-min: Set minimum frequency for scaling. Applies to --pmd-mgmt mode only.
+
+*   --scale-freq-max: Set maximum frequency for scaling. Applies to --pmd-mgmt mode only.
+
 See :doc:`l3_forward` for details.
 The L3fwd-power example reuses the L3fwd command line options.
 
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 20e5b59af9..887c6eae3f 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -265,6 +265,10 @@ static struct rte_eth_conf port_conf = {
 };
 
 static uint32_t max_pkt_len;
+static uint32_t max_empty_polls = 512;
+static uint32_t pause_duration = 1;
+static uint32_t scale_freq_min;
+static uint32_t scale_freq_max;
 
 static struct rte_mempool * pktmbuf_pool[NB_SOCKETS];
 
@@ -1626,10 +1630,32 @@ print_usage(const char *prgname)
 		" empty polls, full polls, and core busyness to telemetry\n"
 		" --interrupt-only: enable interrupt-only mode\n"
 		" --pmd-mgmt MODE: enable PMD power management mode. "
-		"Currently supported modes: baseline, monitor, pause, scale\n",
+		"Currently supported modes: baseline, monitor, pause, scale\n"
+		"  --max-empty-polls MAX_EMPTY_POLLS: number of empty polls to"
+		" wait before entering sleep state\n"
+		"  --pause-duration DURATION: set the duration, in microseconds,"
+		" of the pause callback\n"
+		"  --scale-freq-min FREQ_MIN: set minimum frequency for scaling mode for"
+		" all application lcores (FREQ_MIN must be in kHz, in increments of 100MHz)\n"
+		"  --scale-freq-max FREQ_MAX: set maximum frequency for scaling mode for"
+		" all application lcores (FREQ_MAX must be in kHz, in increments of 100MHz)\n",
 		prgname);
 }
 
+static int
+parse_int(const char *opt)
+{
+	char *end = NULL;
+	unsigned long val;
+
+	/* parse integer string */
+	val = strtoul(opt, &end, 10);
+	if ((opt[0] == '\0') || (end == NULL) || (*end != '\0'))
+		return -1;
+
+	return val;
+}
+
 static int parse_max_pkt_len(const char *pktlen)
 {
 	char *end = NULL;
@@ -1803,6 +1829,10 @@ parse_ep_config(const char *q_arg)
 #define CMD_LINE_OPT_TELEMETRY "telemetry"
 #define CMD_LINE_OPT_PMD_MGMT "pmd-mgmt"
 #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len"
+#define CMD_LINE_OPT_MAX_EMPTY_POLLS "max-empty-polls"
+#define CMD_LINE_OPT_PAUSE_DURATION "pause-duration"
+#define CMD_LINE_OPT_SCALE_FREQ_MIN "scale-freq-min"
+#define CMD_LINE_OPT_SCALE_FREQ_MAX "scale-freq-max"
 
 /* Parse the argument given in the command line of the application */
 static int
@@ -1825,6 +1855,10 @@ parse_args(int argc, char **argv)
 		{CMD_LINE_OPT_TELEMETRY, 0, 0, 0},
 		{CMD_LINE_OPT_INTERRUPT_ONLY, 0, 0, 0},
 		{CMD_LINE_OPT_PMD_MGMT, 1, 0, 0},
+		{CMD_LINE_OPT_MAX_EMPTY_POLLS, 1, 0, 0},
+		{CMD_LINE_OPT_PAUSE_DURATION, 1, 0, 0},
+		{CMD_LINE_OPT_SCALE_FREQ_MIN, 1, 0, 0},
+		{CMD_LINE_OPT_SCALE_FREQ_MAX, 1, 0, 0},
 		{NULL, 0, 0, 0}
 	};
 
@@ -1975,6 +2009,34 @@ parse_args(int argc, char **argv)
 				parse_ptype = 1;
 			}
 
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_MAX_EMPTY_POLLS,
+					sizeof(CMD_LINE_OPT_MAX_EMPTY_POLLS))) {
+				printf("Maximum empty polls configured\n");
+				max_empty_polls = parse_int(optarg);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_PAUSE_DURATION,
+					sizeof(CMD_LINE_OPT_PAUSE_DURATION))) {
+				printf("Pause duration configured\n");
+				pause_duration = parse_int(optarg);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_SCALE_FREQ_MIN,
+					sizeof(CMD_LINE_OPT_SCALE_FREQ_MIN))) {
+				printf("Scaling frequency minimum configured\n");
+				scale_freq_min = parse_int(optarg);
+			}
+
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_SCALE_FREQ_MAX,
+					sizeof(CMD_LINE_OPT_SCALE_FREQ_MAX))) {
+				printf("Scaling frequency maximum configured\n");
+				scale_freq_max = parse_int(optarg);
+			}
+
 			break;
 
 		default:
@@ -2801,6 +2863,28 @@ main(int argc, char **argv)
 			}
 
 			if (app_mode == APP_MODE_PMD_MGMT && !baseline_enabled) {
+				/* Set power_pmd_mgmt configs passed by user */
+				rte_power_pmd_mgmt_set_emptypoll_max(max_empty_polls);
+				ret = rte_power_pmd_mgmt_set_pause_duration(pause_duration);
+				if (ret < 0)
+					rte_exit(EXIT_FAILURE,
+						"Error setting pause_duration: err=%d, lcore=%d\n",
+							ret, lcore_id);
+
+				ret = rte_power_pmd_mgmt_set_scaling_freq_min(lcore_id,
+						scale_freq_min);
+				if (ret < 0)
+					rte_exit(EXIT_FAILURE,
+						"Error setting scaling freq min: err=%d, lcore=%d\n",
+							ret, lcore_id);
+
+				ret = rte_power_pmd_mgmt_set_scaling_freq_max(lcore_id,
+						scale_freq_max);
+				if (ret < 0)
+					rte_exit(EXIT_FAILURE,
+						"Error setting scaling freq max: err=%d, lcore %d\n",
+							ret, lcore_id);
+
 				ret = rte_power_ethdev_pmgmt_queue_enable(
 						lcore_id, portid, queueid,
 						pmgmt_type);
-- 
2.31.1

Re: [PATCH v6 0/4] Add APIs for configurable power options

[Thomas Monjalon]

> Kevin Laatz (4):
>   lib/power: add get and set API for emptypoll max
>   lib/power: add get and set API for pause duration
>   lib/power: add get and set API for scaling freq min and max with
>     pstate mode
>   examples/l3fwd_power: add cli for configurable options

Applied, thanks.