From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 45E66A00C4; Mon, 14 Nov 2022 17:55:26 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 715DB42D37; Mon, 14 Nov 2022 17:54:52 +0100 (CET) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id CBED342D31 for ; Mon, 14 Nov 2022 17:54:49 +0100 (CET) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id EE24221C5DC; Mon, 14 Nov 2022 17:54:48 +0100 (CET) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id 4V5JmXqdFEbj; Mon, 14 Nov 2022 17:54:46 +0100 (CET) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id DD6BF243083; Mon, 14 Nov 2022 17:54:41 +0100 (CET) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com, bruce.richardson@intel.com Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [RFC PATCH v2 05/10] dts: add node memory setup Date: Mon, 14 Nov 2022 16:54:33 +0000 Message-Id: <20221114165438.1133783-6-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20221114165438.1133783-1-juraj.linkes@pantheon.tech> References: <20220824162454.394285-1-juraj.linkes@pantheon.tech> <20221114165438.1133783-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Setup hugepages on nodes. This is useful not only on SUT nodes, but also on TG nodes which use TGs that utilize hugepages. Signed-off-by: Juraj Linkeš --- dts/framework/remote_session/__init__.py | 1 + dts/framework/remote_session/arch/__init__.py | 20 +++++ dts/framework/remote_session/arch/arch.py | 57 +++++++++++++ .../remote_session/os/linux_session.py | 85 +++++++++++++++++++ dts/framework/remote_session/os/os_session.py | 10 +++ dts/framework/testbed_model/node/node.py | 15 +++- 6 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 dts/framework/remote_session/arch/__init__.py create mode 100644 dts/framework/remote_session/arch/arch.py diff --git a/dts/framework/remote_session/__init__.py b/dts/framework/remote_session/__init__.py index f2339b20bd..f0deeadac6 100644 --- a/dts/framework/remote_session/__init__.py +++ b/dts/framework/remote_session/__init__.py @@ -11,4 +11,5 @@ # pylama:ignore=W0611 +from .arch import Arch, create_arch from .os import OSSession, create_session diff --git a/dts/framework/remote_session/arch/__init__.py b/dts/framework/remote_session/arch/__init__.py new file mode 100644 index 0000000000..d78ad42ac5 --- /dev/null +++ b/dts/framework/remote_session/arch/__init__.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2022 PANTHEON.tech s.r.o. + +from framework.config import Architecture, NodeConfiguration + +from .arch import PPC64, Arch, Arm64, i686, x86_32, x86_64 + + +def create_arch(node_config: NodeConfiguration) -> Arch: + match node_config.arch: + case Architecture.x86_64: + return x86_64() + case Architecture.x86_32: + return x86_32() + case Architecture.i686: + return i686() + case Architecture.ppc64le: + return PPC64() + case Architecture.arm64: + return Arm64() diff --git a/dts/framework/remote_session/arch/arch.py b/dts/framework/remote_session/arch/arch.py new file mode 100644 index 0000000000..05c7602def --- /dev/null +++ b/dts/framework/remote_session/arch/arch.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2022 PANTHEON.tech s.r.o. + + +class Arch(object): + """ + Stores architecture-specific information. + """ + + @property + def default_hugepage_memory(self) -> int: + """ + Return the default amount of memory allocated for hugepages DPDK will use. + The default is an amount equal to 256 2MB hugepages (512MB memory). + """ + return 256 * 2048 + + @property + def hugepage_force_first_numa(self) -> bool: + """ + An architecture may need to force configuration of hugepages to first socket. + """ + return False + + +class x86_64(Arch): + @property + def default_hugepage_memory(self) -> int: + return 4096 * 2048 + + +class x86_32(Arch): + @property + def hugepage_force_first_numa(self) -> bool: + return True + + +class i686(Arch): + @property + def default_hugepage_memory(self) -> int: + return 512 * 2048 + + @property + def hugepage_force_first_numa(self) -> bool: + return True + + +class PPC64(Arch): + @property + def default_hugepage_memory(self) -> int: + return 512 * 2048 + + +class Arm64(Arch): + @property + def default_hugepage_memory(self) -> int: + return 2048 * 2048 diff --git a/dts/framework/remote_session/os/linux_session.py b/dts/framework/remote_session/os/linux_session.py index 21f117b714..fad33d7613 100644 --- a/dts/framework/remote_session/os/linux_session.py +++ b/dts/framework/remote_session/os/linux_session.py @@ -3,6 +3,8 @@ # Copyright(c) 2022 University of New Hampshire from framework.config import CPU +from framework.exception import RemoteCommandExecutionError +from framework.utils import expand_range from .posix_session import PosixSession @@ -24,3 +26,86 @@ def get_remote_cpus(self, bypass_core0: bool) -> list[CPU]: continue cpus.append(CPU(int(cpu), int(core), int(socket), int(node))) return cpus + + def setup_hugepages( + self, hugepage_amount: int = -1, force_first_numa: bool = False + ) -> None: + self.logger.info("Getting Hugepage information.") + hugepage_size = self._get_hugepage_size() + hugepages_total = self._get_hugepages_total() + self._numa_nodes = self._get_numa_nodes() + + target_hugepages_total = int(hugepage_amount / hugepage_size) + if hugepage_amount % hugepage_size: + target_hugepages_total += 1 + if force_first_numa or hugepages_total != target_hugepages_total: + # when forcing numa, we need to clear existing hugepages regardless + # of size, so they can be moved to the first numa node + self._configure_huge_pages( + target_hugepages_total, hugepage_size, force_first_numa + ) + else: + self.logger.info("Hugepages already configured.") + self._mount_huge_pages() + + def _get_hugepage_size(self) -> int: + hugepage_size = self.remote_session.send_command( + "awk '/Hugepagesize/ {print $2}' /proc/meminfo" + ).stdout + return int(hugepage_size) + + def _get_hugepages_total(self) -> int: + hugepages_total = self.remote_session.send_command( + "awk '/HugePages_Total/ { print $2 }' /proc/meminfo" + ).stdout + return int(hugepages_total) + + def _get_numa_nodes(self) -> list[int]: + try: + numa_range = self.remote_session.send_command( + "cat /sys/devices/system/node/online", verify=True + ).stdout + numa_range = expand_range(numa_range) + except RemoteCommandExecutionError: + # the file doesn't exist, meaning the node doesn't support numa + numa_range = [] + return numa_range + + def _mount_huge_pages(self) -> None: + self.logger.info("Re-mounting Hugepages.") + hugapge_fs_cmd = "awk '/hugetlbfs/ { print $2 }' /proc/mounts" + self.remote_session.send_command(f"umount $({hugapge_fs_cmd})") + result = self.remote_session.send_command(hugapge_fs_cmd) + if result.stdout == "": + remote_mount_path = "/mnt/huge" + self.remote_session.send_command(f"mkdir -p {remote_mount_path}") + self.remote_session.send_command( + f"mount -t hugetlbfs nodev {remote_mount_path}" + ) + + def _supports_numa(self) -> bool: + # the system supports numa if self._numa_nodes is non-empty and there are more + # than one numa node (in the latter case it may actually support numa, but + # there's no reason to do any numa specific configuration) + return len(self._numa_nodes) > 1 + + def _configure_huge_pages( + self, amount: int, size: int, force_first_numa: bool + ) -> None: + self.logger.info("Configuring Hugepages.") + hugepage_config_path = ( + f"/sys/kernel/mm/hugepages/hugepages-{size}kB/nr_hugepages" + ) + if force_first_numa and self._supports_numa(): + # clear non-numa hugepages + self.remote_session.send_command( + f"echo 0 | sudo tee {hugepage_config_path}" + ) + hugepage_config_path = ( + f"/sys/devices/system/node/node{self._numa_nodes[0]}/hugepages" + f"/hugepages-{size}kB/nr_hugepages" + ) + + self.remote_session.send_command( + f"echo {amount} | sudo tee {hugepage_config_path}" + ) diff --git a/dts/framework/remote_session/os/os_session.py b/dts/framework/remote_session/os/os_session.py index 6f6b6a979e..f84f3ce63c 100644 --- a/dts/framework/remote_session/os/os_session.py +++ b/dts/framework/remote_session/os/os_session.py @@ -144,3 +144,13 @@ def kill_cleanup_dpdk_apps(self, dpdk_prefix_list: Iterable[str]) -> None: Kill and cleanup all DPDK apps identified by dpdk_prefix_list. If dpdk_prefix_list is empty, attempt to find running DPDK apps to kill and clean. """ + + @abstractmethod + def setup_hugepages( + self, hugepage_amount: int = -1, force_first_numa: bool = False + ) -> None: + """ + Get the node's Hugepage Size, configure the specified amount of hugepages + if needed and mount the hugepages if needed. + If force_first_numa is True, configure hugepages just on the first socket. + """ diff --git a/dts/framework/testbed_model/node/node.py b/dts/framework/testbed_model/node/node.py index 5ee7023335..96a1724f4c 100644 --- a/dts/framework/testbed_model/node/node.py +++ b/dts/framework/testbed_model/node/node.py @@ -16,7 +16,7 @@ ) from framework.exception import NodeCleanupError, NodeSetupError, convert_exception from framework.logger import DTSLOG, getLogger -from framework.remote_session import OSSession, create_session +from framework.remote_session import Arch, OSSession, create_arch, create_session from framework.testbed_model.hw import CPUAmount, cpu_filter @@ -33,6 +33,7 @@ class Node(object): config: NodeConfiguration cpus: list[CPU] _other_sessions: list[OSSession] + _arch: Arch def __init__(self, node_config: NodeConfiguration): self.config = node_config @@ -42,6 +43,7 @@ def __init__(self, node_config: NodeConfiguration): self.logger = getLogger(self.name) self.logger.info(f"Created node: {self.name}") self.main_session = create_session(self.config, self.name, self.logger) + self._arch = create_arch(self.config) self._get_remote_cpus() @convert_exception(NodeSetupError) @@ -50,6 +52,7 @@ def setup_execution(self, execution_config: ExecutionConfiguration) -> None: Perform the execution setup that will be done for each execution this node is part of. """ + self._setup_hugepages() self._setup_execution(execution_config) def _setup_execution(self, execution_config: ExecutionConfiguration) -> None: @@ -145,6 +148,16 @@ def _get_remote_cpus(self) -> None: self.logger.info("Getting CPU information.") self.cpus = self.main_session.get_remote_cpus(self.config.bypass_core0) + def _setup_hugepages(self): + """ + Setup hugepages on the Node. Different architectures can supply different + amounts of memory for hugepages and numa-based hugepage allocation may need + to be considered. + """ + self.main_session.setup_hugepages( + self._arch.default_hugepage_memory, self._arch.hugepage_force_first_numa + ) + def close(self) -> None: """ Close all connections and free other resources. -- 2.30.2