[PATCH v1 2/4] l3fwd-power: add option to call uncore api

[tadhgkearney <tadhg.kearney@intel.com>]

Add option for setting uncore frequency min/max/index, through uncore api.
This will be set for each package and die on the SKU. On exit, uncore frequency will be reverted back to previous frequency.

Signed-off-by: tadhgkearney <tadhg.kearney@intel.com>
---
 .../sample_app_ug/l3_forward_power_man.rst    |  28 ++++
 examples/l3fwd-power/main.c                   | 154 +++++++++++++++++-
 2 files changed, 181 insertions(+), 1 deletion(-)

diff --git a/doc/guides/sample_app_ug/l3_forward_power_man.rst b/doc/guides/sample_app_ug/l3_forward_power_man.rst
index 8f6d906200..7b9d99ef54 100644
--- a/doc/guides/sample_app_ug/l3_forward_power_man.rst
+++ b/doc/guides/sample_app_ug/l3_forward_power_man.rst
@@ -97,6 +97,12 @@ where,
 *   -P: Sets all ports to promiscuous mode so that packets are accepted regardless of the packet's Ethernet MAC destination address.
     Without this option, only packets with the Ethernet MAC destination address set to the Ethernet address of the port are accepted.
 
+*   -u: optional, sets uncore frequency to minimum value.
+
+*   -U: optional, sets uncore frequency to maximum value.
+
+*   -i (frequency index): optional, sets uncore frequency to frequency index value, by setting min and max values to be the same.
+
 *   --config (port,queue,lcore)[,(port,queue,lcore)]: determines which queues from which ports are mapped to which cores.
 
 *   --max-pkt-len: optional, maximum packet length in decimal (64-9600)
@@ -364,3 +370,25 @@ in the DPDK Programmer's Guide for more details on PMD power management.
 .. code-block:: console
 
         ./<build_dir>/examples/dpdk-l3fwd-power -l 1-3 -- -p 0x0f --config="(0,0,2),(0,1,3)" --pmd-mgmt=scale
+
+Setting Uncore Values
+---------------------
+
+Uncore frequency can be adjusted through manipulating related sysfs entries to adjust the minimum and maximum uncore values. 
+This will be set for each package and die on the SKU. Three options are available for setting uncore frequency:
+
+``-u``
+  This will set uncore to minimum frequency possible.
+
+``-U``
+  This will set uncore to maximum frequency possible.
+
+``-i``
+  This will allow you to set the specific uncore frequency index that you want, by setting
+  minimum and maximum values to be the same. Frequency index's are set 100000Hz apart from 
+  maximum to minimum.
+  Frequency index values are in decending order, ie, index 0 is maximum frequency index.
+
+.. code-block:: console
+
+        ./<build_dir>/examples/dpdk-l3fwd-power -l 1-3 -- -p 0x0f --config="(0,0,2),(0,1,3)" -i 1
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 887c6eae3f..98a2e8428a 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -15,6 +15,8 @@
 #include <unistd.h>
 #include <signal.h>
 #include <math.h>
+#include <dirent.h>
+#include <fnmatch.h>
 
 #include <rte_common.h>
 #include <rte_byteorder.h>
@@ -47,6 +49,7 @@
 #include <rte_metrics.h>
 #include <rte_telemetry.h>
 #include <rte_power_pmd_mgmt.h>
+#include <rte_power_uncore.h>
 
 #include "perf_core.h"
 #include "main.h"
@@ -71,6 +74,7 @@
 
 #ifndef APP_LOOKUP_METHOD
 #define APP_LOOKUP_METHOD             APP_LOOKUP_LPM
+
 #endif
 
 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
@@ -134,9 +138,14 @@
 
 #define NUM_TELSTATS RTE_DIM(telstats_strings)
 
+#define UNCORE_FREQUENCY_DIR "/sys/devices/system/cpu/intel_uncore_frequency"
+
 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
 
+/* Max number of nodes times dies available on uncore */
+#define MAX_DIE_NODES (RTE_MAX_NUMA_DIE * RTE_MAX_NUMA_NODES)
+
 /* ethernet addresses of ports */
 static struct rte_ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
 
@@ -145,6 +154,8 @@ static rte_spinlock_t locks[RTE_MAX_ETHPORTS];
 
 /* mask of enabled ports */
 static uint32_t enabled_port_mask = 0;
+/* if uncore frequency was enabled without errors */
+static int enabled_uncore = 0;
 /* Ports set in promiscuous mode off by default. */
 static int promiscuous_on = 0;
 /* NUMA is enabled by default. */
@@ -165,6 +176,13 @@ struct telstats_name {
 	char name[RTE_ETH_XSTATS_NAME_SIZE];
 };
 
+struct uncore_info{
+	unsigned int pkg;
+	unsigned int die;
+};
+
+struct uncore_info ui[MAX_DIE_NODES];
+
 /* telemetry stats to be reported */
 const struct telstats_name telstats_strings[] = {
 	{"empty_poll"},
@@ -1616,6 +1634,9 @@ print_usage(const char *prgname)
 		"  [--max-pkt-len PKTLEN]\n"
 		"  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
 		"  -P: enable promiscuous mode\n"
+		"  -u: set min frequency for uncore\n"
+		"  -U: set max frequency for uncore\n"
+		"  -i (frequency index): set frequency index for uncore\n"
 		"  --config (port,queue,lcore): rx queues configuration\n"
 		"  --high-perf-cores CORELIST: list of high performance cores\n"
 		"  --perf-config: similar as config, cores specified as indices"
@@ -1672,6 +1693,106 @@ static int parse_max_pkt_len(const char *pktlen)
 	return len;
 }
 
+static int
+parse_uncore_dir(void)
+{
+	DIR *d;
+	struct dirent *dir;
+	char pkg[3], die[3];
+	unsigned int count = 0;
+	const char filter[] = "*freq_khz*";
+	
+	d = opendir(UNCORE_FREQUENCY_DIR);
+ 	if (!d) {
+ 		RTE_LOG(ERR, EAL, "Unable to open uncore frequency directory");
+		return -1;
+	}
+
+	else {
+		/* Loop through every file name in uncore frequency dir extracting pkg + die numbers from file names */
+    	while ((dir = readdir(d)) != NULL) {
+			if (fnmatch(filter, dir->d_name, 0) > 0) {
+				/* Extract pkg + die numbers from string 'package_XX_die_XX' */
+				sprintf(pkg, "%c%c", dir->d_name[8], dir->d_name[9]);
+				sprintf(die, "%c%c", dir->d_name[15], dir->d_name[16]);
+				ui[count].pkg = atoi(pkg);
+				ui[count].die = atoi(die);
+				count++;
+			}
+    	}
+    	closedir(d);
+  	}
+  	return(count);
+}
+
+static int
+parse_uncore_min_max(const char *choice)
+{
+	int ret = 0, total_files, i;
+	unsigned int die, pkg;
+	total_files = parse_uncore_dir();
+
+	if (total_files < 1)
+		return -1;
+
+	for (i = 0; i < total_files; i++){
+		die = ui[i].die;
+		pkg = ui[i].pkg;
+		rte_power_uncore_init(pkg, die);
+		if (!strncmp(choice, "max", 3)){
+			ret = rte_power_uncore_freq_max(pkg, die);
+			if (ret == -1)
+				RTE_LOG(INFO, L3FWD_POWER, "Unable to set max uncore value for pkg %02u die %02u\n", pkg, die);
+		}
+
+		else if (!strncmp(choice, "min", 3)){
+			ret = rte_power_uncore_freq_min(pkg, die);
+			if (ret == -1){
+				RTE_LOG(INFO, L3FWD_POWER, "Unable to set min uncore value for pkg %02u die %02u\n", pkg, die);
+				return ret;
+			}
+		}
+		else {
+			RTE_LOG(INFO, L3FWD_POWER, "Uncore choice provided invalid\n");
+			return -1;
+		}		
+	}
+
+	RTE_LOG(INFO, L3FWD_POWER, "Successfully set max/min uncore frequency.\n");
+	return ret;
+}
+
+static int
+parse_uncore_index(const char *choice)
+{
+	int ret = 0, total_files, i;
+	unsigned int  die, pkg;
+	total_files = parse_uncore_dir();
+
+	if (total_files < 1)
+		return -1;
+
+	for (i = 0; i < total_files; i++){
+		die = ui[i].die;
+		pkg = ui[i].pkg;
+		rte_power_uncore_init(pkg, die);
+
+		int frequency_index= atoi(choice);
+		int freq_array_len = rte_power_uncore_get_num_freqs(pkg, die);
+		if (frequency_index > freq_array_len){
+			RTE_LOG(INFO, L3FWD_POWER, "Frequency index given out of range, please choose a value from 0 to %d.\n", freq_array_len);
+			return -1;
+		}
+		ret = rte_power_set_uncore_freq(pkg, die, frequency_index);
+		if (ret == -1){
+			RTE_LOG(INFO, L3FWD_POWER, "Unable to set uncore index value for pkg %02u die %02u\n", pkg, die);
+			return ret;
+		}
+	}
+	RTE_LOG(INFO, L3FWD_POWER, "Successfully set freq index for uncore\n");	
+	return ret;
+}
+
 static int
 parse_portmask(const char *portmask)
 {
@@ -1864,7 +1985,7 @@ parse_args(int argc, char **argv)
 
 	argvopt = argv;
 
-	while ((opt = getopt_long(argc, argvopt, "p:l:m:h:P",
+	while ((opt = getopt_long(argc, argvopt, "p:l:m:h:P:u U i:",
 				lgopts, &option_index)) != EOF) {
 
 		switch (opt) {
@@ -1893,6 +2014,32 @@ parse_args(int argc, char **argv)
 			limit = parse_max_pkt_len(optarg);
 			freq_tlb[HGH] = limit;
 			break;
+		case 'u': ;
+			const char* min = "min";
+			enabled_uncore = parse_uncore_min_max(min);
+			if (enabled_uncore < 0) {
+				printf("Unable to reach min uncore value\n");
+				print_usage(prgname);
+				return -1;
+			}
+			break;
+		case 'U': ;
+			const char* max = "max";
+			enabled_uncore = parse_uncore_min_max(max);
+			if (enabled_uncore < 0) {
+				printf("Unable to reach max uncore value\n");
+				print_usage(prgname);
+				return -1;
+			}
+			break;
+		case 'i':
+			enabled_uncore = parse_uncore_index(optarg);
+			if (enabled_uncore < 0) {
+				printf("Unable to reach uncore index value\n");
+				print_usage(prgname);
+				return -1;
+			}
+			break;
 		/* long options */
 		case 0:
 			if (!strncmp(lgopts[option_index].name, "config", 6)) {
@@ -2365,6 +2512,8 @@ static int
 deinit_power_library(void)
 {
 	unsigned int lcore_id;
+	int i, total_files;
+	total_files = parse_uncore_dir();
 	int ret = 0;
 
 	RTE_LCORE_FOREACH(lcore_id) {
@@ -2377,6 +2526,9 @@ deinit_power_library(void)
 			return ret;
 		}
 	}
+
+	for (i = 0; i < total_files; i++)
+		rte_power_uncore_exit(ui[i].pkg, ui[i].die);
 	return ret;
 }
 
-- 
2.25.1