Re: [PATCH] dts: Change hugepage runtime config to 2MB Exclusively

[Patrick Robb <probb@iol.unh.edu>]

Recheck-request: iol-intel-Functional

checking for lab infra failure

On Thu, Apr 4, 2024 at 11:31 AM Nicholas Pratte <npratte@iol.unh.edu> wrote:
>
> The previous implementation configures and allocates hugepage sizes
> based on a system default. This can lead to two problems: overallocation of
> hugepages (which may crash the remote host), and configuration of hugepage
> sizes that are not recommended during runtime. This new implementation
> allows only 2MB hugepage allocation during runtime; any other unique
> hugepage size must be configured by the end-user for initializing DTS.
>
> If the amount of 2MB hugepages requested exceeds the amount of 2MB
> hugepages already configured on the system, then the system will remount
> hugepages to cover the difference. If the amount of hugepages requested is
> either less than or equal to the amount already configured on the system,
> then nothing is done.
>
> Bugzilla ID: 1370
> Signed-off-by: Nicholas Pratte <npratte@iol.unh.edu>
> Reviewed-by: Jeremy Spewock <jspewock@iol.unh.edu>
> ---
>  dts/conf.yaml                                |  8 ++++----
>  dts/framework/config/__init__.py             |  4 ++--
>  dts/framework/config/conf_yaml_schema.json   |  6 +++---
>  dts/framework/config/types.py                |  2 +-
>  dts/framework/testbed_model/linux_session.py | 21 +++++++++-----------
>  5 files changed, 19 insertions(+), 22 deletions(-)
>
> diff --git a/dts/conf.yaml b/dts/conf.yaml
> index 8068345dd5..3690e77ee5 100644
> --- a/dts/conf.yaml
> +++ b/dts/conf.yaml
> @@ -35,8 +35,8 @@ nodes:
>      lcores: "" # use all the available logical cores
>      use_first_core: false # tells DPDK to use any physical core
>      memory_channels: 4 # tells DPDK to use 4 memory channels
> -    hugepages:  # optional; if removed, will use system hugepage configuration
> -        amount: 256
> +    hugepages_2mb: # optional; if removed, will use system hugepage configuration
> +        amount: 2560
>          force_first_numa: false
>      ports:
>        # sets up the physical link between "SUT 1"@000:00:08.0 and "TG 1"@0000:00:08.0
> @@ -71,8 +71,8 @@ nodes:
>          os_driver: rdma
>          peer_node: "SUT 1"
>          peer_pci: "0000:00:08.1"
> -    hugepages:  # optional; if removed, will use system hugepage configuration
> -        amount: 256
> +    hugepages_2mb: # optional; if removed, will use system hugepage configuration
> +        amount: 2560
>          force_first_numa: false
>      traffic_generator:
>          type: SCAPY
> diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py
> index 4cb5c74059..b6f820e39e 100644
> --- a/dts/framework/config/__init__.py
> +++ b/dts/framework/config/__init__.py
> @@ -255,8 +255,8 @@ def from_dict(
>              Either an SUT or TG configuration instance.
>          """
>          hugepage_config = None
> -        if "hugepages" in d:
> -            hugepage_config_dict = d["hugepages"]
> +        if "hugepages_2mb" in d:
> +            hugepage_config_dict = d["hugepages_2mb"]
>              if "force_first_numa" not in hugepage_config_dict:
>                  hugepage_config_dict["force_first_numa"] = False
>              hugepage_config = HugepageConfiguration(**hugepage_config_dict)
> diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json
> index 4731f4511d..f4d7199523 100644
> --- a/dts/framework/config/conf_yaml_schema.json
> +++ b/dts/framework/config/conf_yaml_schema.json
> @@ -146,7 +146,7 @@
>          "compiler"
>        ]
>      },
> -    "hugepages": {
> +    "hugepages_2mb": {
>        "type": "object",
>        "description": "Optional hugepage configuration. If not specified, hugepages won't be configured and DTS will use system configuration.",
>        "properties": {
> @@ -253,8 +253,8 @@
>              "type": "integer",
>              "description": "How many memory channels to use. Optional, defaults to 1."
>            },
> -          "hugepages": {
> -            "$ref": "#/definitions/hugepages"
> +          "hugepages_2mb": {
> +            "$ref": "#/definitions/hugepages_2mb"
>            },
>            "ports": {
>              "type": "array",
> diff --git a/dts/framework/config/types.py b/dts/framework/config/types.py
> index 1927910d88..016e0c3dbd 100644
> --- a/dts/framework/config/types.py
> +++ b/dts/framework/config/types.py
> @@ -46,7 +46,7 @@ class NodeConfigDict(TypedDict):
>      """Allowed keys and values."""
>
>      #:
> -    hugepages: HugepageConfigurationDict
> +    hugepages_2mb: HugepageConfigurationDict
>      #:
>      name: str
>      #:
> diff --git a/dts/framework/testbed_model/linux_session.py b/dts/framework/testbed_model/linux_session.py
> index 5d24030c3d..ad3c811301 100644
> --- a/dts/framework/testbed_model/linux_session.py
> +++ b/dts/framework/testbed_model/linux_session.py
> @@ -15,7 +15,7 @@
>
>  from typing_extensions import NotRequired
>
> -from framework.exception import RemoteCommandExecutionError
> +from framework.exception import ConfigurationError, RemoteCommandExecutionError
>  from framework.utils import expand_range
>
>  from .cpu import LogicalCore
> @@ -87,25 +87,22 @@ def get_dpdk_file_prefix(self, dpdk_prefix: str) -> str:
>      def setup_hugepages(self, hugepage_count: int, force_first_numa: bool) -> None:
>          """Overrides :meth:`~.os_session.OSSession.setup_hugepages`."""
>          self._logger.info("Getting Hugepage information.")
> -        hugepage_size = self._get_hugepage_size()
> +        if "hugepages-2048kB" not in self.send_command("ls /sys/kernel/mm/hugepages").stdout:
> +            raise ConfigurationError("2MB hugepages not supported by operating system")
>          hugepages_total = self._get_hugepages_total()
>          self._numa_nodes = self._get_numa_nodes()
>
> -        if force_first_numa or hugepages_total != hugepage_count:
> +        if force_first_numa or hugepages_total < hugepage_count:
>              # when forcing numa, we need to clear existing hugepages regardless
>              # of size, so they can be moved to the first numa node
> -            self._configure_huge_pages(hugepage_count, hugepage_size, force_first_numa)
> +            self._configure_huge_pages(hugepage_count, force_first_numa)
>          else:
>              self._logger.info("Hugepages already configured.")
>          self._mount_huge_pages()
>
> -    def _get_hugepage_size(self) -> int:
> -        hugepage_size = self.send_command("awk '/Hugepagesize/ {print $2}' /proc/meminfo").stdout
> -        return int(hugepage_size)
> -
>      def _get_hugepages_total(self) -> int:
>          hugepages_total = self.send_command(
> -            "awk '/HugePages_Total/ { print $2 }' /proc/meminfo"
> +            "cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
>          ).stdout
>          return int(hugepages_total)
>
> @@ -136,15 +133,15 @@ def _supports_numa(self) -> bool:
>          # there's no reason to do any numa specific configuration)
>          return len(self._numa_nodes) > 1
>
> -    def _configure_huge_pages(self, amount: int, size: int, force_first_numa: bool) -> None:
> +    def _configure_huge_pages(self, amount: int, force_first_numa: bool) -> None:
>          self._logger.info("Configuring Hugepages.")
> -        hugepage_config_path = f"/sys/kernel/mm/hugepages/hugepages-{size}kB/nr_hugepages"
> +        hugepage_config_path = "/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
>          if force_first_numa and self._supports_numa():
>              # clear non-numa hugepages
>              self.send_command(f"echo 0 | tee {hugepage_config_path}", privileged=True)
>              hugepage_config_path = (
>                  f"/sys/devices/system/node/node{self._numa_nodes[0]}/hugepages"
> -                f"/hugepages-{size}kB/nr_hugepages"
> +                f"/hugepages-2048kB/nr_hugepages"
>              )
>
>          self.send_command(f"echo {amount} | tee {hugepage_config_path}", privileged=True)
> --
> 2.44.0
>