From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id E9B2FA0566; Wed, 16 Nov 2022 14:48:31 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 9E59F40E03; Wed, 16 Nov 2022 14:48:31 +0100 (CET) Received: from mail-pf1-f174.google.com (mail-pf1-f174.google.com [209.85.210.174]) by mails.dpdk.org (Postfix) with ESMTP id 01D1240DFB for ; Wed, 16 Nov 2022 14:48:29 +0100 (CET) Received: by mail-pf1-f174.google.com with SMTP id k15so17542623pfg.2 for ; Wed, 16 Nov 2022 05:48:29 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=iol.unh.edu; s=unh-iol; h=cc:to:subject:message-id:date:from:in-reply-to:references :mime-version:from:to:cc:subject:date:message-id:reply-to; bh=8DAwj1zb+WHuvIZ8qiiiiudrWYrhilP/V+tvllTOAEM=; b=YBtqu1bO5PKceUM0JkDYneqKcb8gdrCRWSC3jc9SX75S19Qo9Trm81X0qoCRLhJ3tA pRXDlmWmGZs94/IAsYzScJb/+VaFzyZMGAnsa23hj9c/ME1RlSrgFUZE+zeRW7SYmxhC xDqgzzhF70t8kzgR5/qCy7oAQ08r6twa/SDBQ= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=cc:to:subject:message-id:date:from:in-reply-to:references :mime-version:x-gm-message-state:from:to:cc:subject:date:message-id :reply-to; bh=8DAwj1zb+WHuvIZ8qiiiiudrWYrhilP/V+tvllTOAEM=; b=XSI+o2oIMpxI5yFlbKvNuGAlNUkoYL4Oszm3zAQ0tE/vB4YCsWK8d5YoGDZHTi0ILV Jj3SWAYOJv18PWmBHSt8mibHp/SljxiXLcLVptLzZSsL63sx3ckcE6atH7g9nkdgyKc/ 5Q4TcOl1uxB1fC9cOS3nf7B4z23YK6HhAXhiF37+nZLlTn9bqQwKfEgevI4VybOzdNLP vZgD5+eOiVwkwKvDHa61KsSaqR5G3A2+/rhr/t/FSK5fmWZWKWerNMDHfrxtdd2LbNbK lKC0tiBRDDucRzfPfsHQIQt1HWU/6iz+2hKiboNqtSL5m9z8V4FYB9mvuk6G227LSJAf NC1w== X-Gm-Message-State: ANoB5pkAjFJWWFq4GNe+Brfs1w1qWukUTl+aogyuG5ssQnctNUPtHUwl lUSGnRTYL5Tig/ad6aa3tfrN6HZTrOxSv4TBsCsV0g== X-Google-Smtp-Source: AA0mqf4q1eGe3rotf3H3z6u2uSsGMmLE1Yf6a2j/2IX2y9tJ+RtWc7UZbZZ0RtFFbLSWpJ1+m7O3kkdW7dQeYjBnmpg= X-Received: by 2002:a05:6a00:1d05:b0:56c:4303:a93d with SMTP id a5-20020a056a001d0500b0056c4303a93dmr23070256pfx.73.1668606509058; Wed, 16 Nov 2022 05:48:29 -0800 (PST) MIME-Version: 1.0 References: <20220824162454.394285-1-juraj.linkes@pantheon.tech> <20221114165438.1133783-1-juraj.linkes@pantheon.tech> <20221114165438.1133783-6-juraj.linkes@pantheon.tech> In-Reply-To: <20221114165438.1133783-6-juraj.linkes@pantheon.tech> From: Owen Hilyard Date: Wed, 16 Nov 2022 08:47:53 -0500 Message-ID: Subject: Re: [RFC PATCH v2 05/10] dts: add node memory setup To: =?UTF-8?Q?Juraj_Linke=C5=A1?= Cc: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, lijuan.tu@intel.com, bruce.richardson@intel.com, dev@dpdk.org Content-Type: multipart/alternative; boundary="0000000000003e28fd05ed96ba33" X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org --0000000000003e28fd05ed96ba33 Content-Type: text/plain; charset="UTF-8" Content-Transfer-Encoding: quoted-printable On Mon, Nov 14, 2022 at 11:54 AM Juraj Linke=C5=A1 wrote: > Setup hugepages on nodes. This is useful not only on SUT nodes, but > also on TG nodes which use TGs that utilize hugepages. > > Signed-off-by: Juraj Linke=C5=A1 > --- > dts/framework/remote_session/__init__.py | 1 + > dts/framework/remote_session/arch/__init__.py | 20 +++++ > dts/framework/remote_session/arch/arch.py | 57 +++++++++++++ > .../remote_session/os/linux_session.py | 85 +++++++++++++++++++ > dts/framework/remote_session/os/os_session.py | 10 +++ > dts/framework/testbed_model/node/node.py | 15 +++- > 6 files changed, 187 insertions(+), 1 deletion(-) > create mode 100644 dts/framework/remote_session/arch/__init__.py > create mode 100644 dts/framework/remote_session/arch/arch.py > > diff --git a/dts/framework/remote_session/__init__.py > b/dts/framework/remote_session/__init__.py > index f2339b20bd..f0deeadac6 100644 > --- a/dts/framework/remote_session/__init__.py > +++ b/dts/framework/remote_session/__init__.py > @@ -11,4 +11,5 @@ > > # pylama:ignore=3DW0611 > > +from .arch import Arch, create_arch > from .os import OSSession, create_session > diff --git a/dts/framework/remote_session/arch/__init__.py > b/dts/framework/remote_session/arch/__init__.py > new file mode 100644 > index 0000000000..d78ad42ac5 > --- /dev/null > +++ b/dts/framework/remote_session/arch/__init__.py > @@ -0,0 +1,20 @@ > +# SPDX-License-Identifier: BSD-3-Clause > +# Copyright(c) 2022 PANTHEON.tech s.r.o. > + > +from framework.config import Architecture, NodeConfiguration > + > +from .arch import PPC64, Arch, Arm64, i686, x86_32, x86_64 > + > + > +def create_arch(node_config: NodeConfiguration) -> Arch: > + match node_config.arch: > + case Architecture.x86_64: > + return x86_64() > + case Architecture.x86_32: > + return x86_32() > + case Architecture.i686: > + return i686() > + case Architecture.ppc64le: > + return PPC64() > + case Architecture.arm64: > + return Arm64() > diff --git a/dts/framework/remote_session/arch/arch.py > b/dts/framework/remote_session/arch/arch.py > new file mode 100644 > index 0000000000..05c7602def > --- /dev/null > +++ b/dts/framework/remote_session/arch/arch.py > @@ -0,0 +1,57 @@ > +# SPDX-License-Identifier: BSD-3-Clause > +# Copyright(c) 2022 PANTHEON.tech s.r.o. > + > + > +class Arch(object): > + """ > + Stores architecture-specific information. > + """ > + > + @property > + def default_hugepage_memory(self) -> int: > + """ > + Return the default amount of memory allocated for hugepages DPDK > will use. > + The default is an amount equal to 256 2MB hugepages (512MB > memory). > + """ > + return 256 * 2048 > + > + @property > + def hugepage_force_first_numa(self) -> bool: > + """ > + An architecture may need to force configuration of hugepages to > first socket. > + """ > + return False > + > + > +class x86_64(Arch): > + @property > + def default_hugepage_memory(self) -> int: > + return 4096 * 2048 > + > + > +class x86_32(Arch): > + @property > + def hugepage_force_first_numa(self) -> bool: > + return True > + > + > +class i686(Arch): > + @property > + def default_hugepage_memory(self) -> int: > + return 512 * 2048 > + > + @property > + def hugepage_force_first_numa(self) -> bool: > + return True > + > + > +class PPC64(Arch): > + @property > + def default_hugepage_memory(self) -> int: > + return 512 * 2048 > + > + > +class Arm64(Arch): > + @property > + def default_hugepage_memory(self) -> int: > + return 2048 * 2048 > diff --git a/dts/framework/remote_session/os/linux_session.py > b/dts/framework/remote_session/os/linux_session.py > index 21f117b714..fad33d7613 100644 > --- a/dts/framework/remote_session/os/linux_session.py > +++ b/dts/framework/remote_session/os/linux_session.py > @@ -3,6 +3,8 @@ > # Copyright(c) 2022 University of New Hampshire > > from framework.config import CPU > +from framework.exception import RemoteCommandExecutionError > +from framework.utils import expand_range > > from .posix_session import PosixSession > > @@ -24,3 +26,86 @@ def get_remote_cpus(self, bypass_core0: bool) -> > list[CPU]: > continue > cpus.append(CPU(int(cpu), int(core), int(socket), int(node))= ) > return cpus > + > + def setup_hugepages( > + self, hugepage_amount: int =3D -1, force_first_numa: bool =3D Fa= lse > I think that hugepage_amount: int | None =3D None is better, since it expresses it is an optional argument and the type checker will force anyone using the value to check if it is none, whereas that will not happen with -1. > + ) -> None: > + self.logger.info("Getting Hugepage information.") > + hugepage_size =3D self._get_hugepage_size() > + hugepages_total =3D self._get_hugepages_total() > + self._numa_nodes =3D self._get_numa_nodes() > + > + target_hugepages_total =3D int(hugepage_amount / hugepage_size) > + if hugepage_amount % hugepage_size: > + target_hugepages_total +=3D 1 > + if force_first_numa or hugepages_total !=3D target_hugepages_tot= al: > + # when forcing numa, we need to clear existing hugepages > regardless > + # of size, so they can be moved to the first numa node > + self._configure_huge_pages( > + target_hugepages_total, hugepage_size, force_first_numa > + ) > + else: > + self.logger.info("Hugepages already configured.") > + self._mount_huge_pages() > + > + def _get_hugepage_size(self) -> int: > + hugepage_size =3D self.remote_session.send_command( > + "awk '/Hugepagesize/ {print $2}' /proc/meminfo" + ).stdout > + return int(hugepage_size) > + > + def _get_hugepages_total(self) -> int: > + hugepages_total =3D self.remote_session.send_command( > + "awk '/HugePages_Total/ { print $2 }' /proc/meminfo" + ).stdout > + return int(hugepages_total) > + > + def _get_numa_nodes(self) -> list[int]: > + try: > + numa_range =3D self.remote_session.send_command( > + "cat /sys/devices/system/node/online", verify=3DTrue + ).stdout > + numa_range =3D expand_range(numa_range) > + except RemoteCommandExecutionError: > + # the file doesn't exist, meaning the node doesn't support > numa > + numa_range =3D [] > + return numa_range > + > + def _mount_huge_pages(self) -> None: > + self.logger.info("Re-mounting Hugepages.") > + hugapge_fs_cmd =3D "awk '/hugetlbfs/ { print $2 }' /proc/mounts" + self.remote_session.send_command(f"umount $({hugapge_fs_cmd})") > + result =3D self.remote_session.send_command(hugapge_fs_cmd) > + if result.stdout =3D=3D "": > + remote_mount_path =3D "/mnt/huge" > + self.remote_session.send_command(f"mkdir -p > {remote_mount_path}") > + self.remote_session.send_command( > + f"mount -t hugetlbfs nodev {remote_mount_path}" > + ) > + > + def _supports_numa(self) -> bool: > + # the system supports numa if self._numa_nodes is non-empty and > there are more > + # than one numa node (in the latter case it may actually support > numa, but > + # there's no reason to do any numa specific configuration) > + return len(self._numa_nodes) > 1 > + > + def _configure_huge_pages( > + self, amount: int, size: int, force_first_numa: bool > + ) -> None: + self.logger.info("Configuring Hugepages.") > + hugepage_config_path =3D ( > + f"/sys/kernel/mm/hugepages/hugepages-{size}kB/nr_hugepages" > + ) > + if force_first_numa and self._supports_numa(): > + # clear non-numa hugepages > + self.remote_session.send_command( > + f"echo 0 | sudo tee {hugepage_config_path}" > + ) > + hugepage_config_path =3D ( > + > f"/sys/devices/system/node/node{self._numa_nodes[0]}/hugepages" > + f"/hugepages-{size}kB/nr_hugepages" > + ) > + > + self.remote_session.send_command( > + f"echo {amount} | sudo tee {hugepage_config_path}" > + ) > diff --git a/dts/framework/remote_session/os/os_session.py > b/dts/framework/remote_session/os/os_session.py > index 6f6b6a979e..f84f3ce63c 100644 > --- a/dts/framework/remote_session/os/os_session.py > +++ b/dts/framework/remote_session/os/os_session.py > @@ -144,3 +144,13 @@ def kill_cleanup_dpdk_apps(self, dpdk_prefix_list: > Iterable[str]) -> None: > Kill and cleanup all DPDK apps identified by dpdk_prefix_list. I= f > dpdk_prefix_list is empty, attempt to find running DPDK apps to > kill and clean. > """ > + > + @abstractmethod > + def setup_hugepages( > + self, hugepage_amount: int =3D -1, force_first_numa: bool =3D Fa= lse > + ) -> None: > + """ > + Get the node's Hugepage Size, configure the specified amount of > hugepages > + if needed and mount the hugepages if needed. > + If force_first_numa is True, configure hugepages just on the > first socket. > + """ > diff --git a/dts/framework/testbed_model/node/node.py > b/dts/framework/testbed_model/node/node.py > index 5ee7023335..96a1724f4c 100644 > --- a/dts/framework/testbed_model/node/node.py > +++ b/dts/framework/testbed_model/node/node.py > @@ -16,7 +16,7 @@ > ) > from framework.exception import NodeCleanupError, NodeSetupError, > convert_exception > from framework.logger import DTSLOG, getLogger > -from framework.remote_session import OSSession, create_session > +from framework.remote_session import Arch, OSSession, create_arch, > create_session > from framework.testbed_model.hw import CPUAmount, cpu_filter > > > @@ -33,6 +33,7 @@ class Node(object): > config: NodeConfiguration > cpus: list[CPU] > _other_sessions: list[OSSession] > + _arch: Arch > > def __init__(self, node_config: NodeConfiguration): > self.config =3D node_config > @@ -42,6 +43,7 @@ def __init__(self, node_config: NodeConfiguration): > self.logger =3D getLogger(self.name) > self.logger.info(f"Created node: {self.name}") > self.main_session =3D create_session(self.config, self.name, > self.logger) > + self._arch =3D create_arch(self.config) > self._get_remote_cpus() > > @convert_exception(NodeSetupError) > @@ -50,6 +52,7 @@ def setup_execution(self, execution_config: > ExecutionConfiguration) -> None: > Perform the execution setup that will be done for each execution > this node is part of. > """ > + self._setup_hugepages() > self._setup_execution(execution_config) > > def _setup_execution(self, execution_config: ExecutionConfiguration) > -> None: > @@ -145,6 +148,16 @@ def _get_remote_cpus(self) -> None: > self.logger.info("Getting CPU information.") > self.cpus =3D > self.main_session.get_remote_cpus(self.config.bypass_core0) > > + def _setup_hugepages(self): > + """ > + Setup hugepages on the Node. Different architectures can supply > different > + amounts of memory for hugepages and numa-based hugepage > allocation may need > + to be considered. > + """ > + self.main_session.setup_hugepages( > + self._arch.default_hugepage_memory, > self._arch.hugepage_force_first_numa > + ) > + > def close(self) -> None: > """ > Close all connections and free other resources. > -- > 2.30.2 > > On Mon, Nov 14, 2022 at 11:54 AM Juraj Linke=C5=A1 wrote: > Setup hugepages on nodes. This is useful not only on SUT nodes, but > also on TG nodes which use TGs that utilize hugepages. > > Signed-off-by: Juraj Linke=C5=A1 > --- > dts/framework/remote_session/__init__.py | 1 + > dts/framework/remote_session/arch/__init__.py | 20 +++++ > dts/framework/remote_session/arch/arch.py | 57 +++++++++++++ > .../remote_session/os/linux_session.py | 85 +++++++++++++++++++ > dts/framework/remote_session/os/os_session.py | 10 +++ > dts/framework/testbed_model/node/node.py | 15 +++- > 6 files changed, 187 insertions(+), 1 deletion(-) > create mode 100644 dts/framework/remote_session/arch/__init__.py > create mode 100644 dts/framework/remote_session/arch/arch.py > > diff --git a/dts/framework/remote_session/__init__.py > b/dts/framework/remote_session/__init__.py > index f2339b20bd..f0deeadac6 100644 > --- a/dts/framework/remote_session/__init__.py > +++ b/dts/framework/remote_session/__init__.py > @@ -11,4 +11,5 @@ > > # pylama:ignore=3DW0611 > > +from .arch import Arch, create_arch > from .os import OSSession, create_session > diff --git a/dts/framework/remote_session/arch/__init__.py > b/dts/framework/remote_session/arch/__init__.py > new file mode 100644 > index 0000000000..d78ad42ac5 > --- /dev/null > +++ b/dts/framework/remote_session/arch/__init__.py > @@ -0,0 +1,20 @@ > +# SPDX-License-Identifier: BSD-3-Clause > +# Copyright(c) 2022 PANTHEON.tech s.r.o. > + > +from framework.config import Architecture, NodeConfiguration > + > +from .arch import PPC64, Arch, Arm64, i686, x86_32, x86_64 > + > + > +def create_arch(node_config: NodeConfiguration) -> Arch: > + match node_config.arch: > + case Architecture.x86_64: > + return x86_64() > + case Architecture.x86_32: > + return x86_32() > + case Architecture.i686: > + return i686() > + case Architecture.ppc64le: > + return PPC64() > + case Architecture.arm64: > + return Arm64() > diff --git a/dts/framework/remote_session/arch/arch.py > b/dts/framework/remote_session/arch/arch.py > new file mode 100644 > index 0000000000..05c7602def > --- /dev/null > +++ b/dts/framework/remote_session/arch/arch.py > @@ -0,0 +1,57 @@ > +# SPDX-License-Identifier: BSD-3-Clause > +# Copyright(c) 2022 PANTHEON.tech s.r.o. > + > + > +class Arch(object): > + """ > + Stores architecture-specific information. > + """ > + > + @property > + def default_hugepage_memory(self) -> int: > + """ > + Return the default amount of memory allocated for hugepages DPDK > will use. > + The default is an amount equal to 256 2MB hugepages (512MB > memory). > + """ > + return 256 * 2048 > + > + @property > + def hugepage_force_first_numa(self) -> bool: > + """ > + An architecture may need to force configuration of hugepages to > first socket. > + """ > + return False > + > + > +class x86_64(Arch): > + @property > + def default_hugepage_memory(self) -> int: > + return 4096 * 2048 > + > + > +class x86_32(Arch): > + @property > + def hugepage_force_first_numa(self) -> bool: > + return True > + > + > +class i686(Arch): > + @property > + def default_hugepage_memory(self) -> int: > + return 512 * 2048 > + > + @property > + def hugepage_force_first_numa(self) -> bool: > + return True > + > + > +class PPC64(Arch): > + @property > + def default_hugepage_memory(self) -> int: > + return 512 * 2048 > + > + > +class Arm64(Arch): > + @property > + def default_hugepage_memory(self) -> int: > + return 2048 * 2048 > diff --git a/dts/framework/remote_session/os/linux_session.py > b/dts/framework/remote_session/os/linux_session.py > index 21f117b714..fad33d7613 100644 > --- a/dts/framework/remote_session/os/linux_session.py > +++ b/dts/framework/remote_session/os/linux_session.py > @@ -3,6 +3,8 @@ > # Copyright(c) 2022 University of New Hampshire > > from framework.config import CPU > +from framework.exception import RemoteCommandExecutionError > +from framework.utils import expand_range > > from .posix_session import PosixSession > > @@ -24,3 +26,86 @@ def get_remote_cpus(self, bypass_core0: bool) -> > list[CPU]: > continue > cpus.append(CPU(int(cpu), int(core), int(socket), int(node))= ) > return cpus > + > + def setup_hugepages( > + self, hugepage_amount: int =3D -1, force_first_numa: bool =3D Fa= lse > + ) -> None: > + self.logger.info("Getting Hugepage information.") > + hugepage_size =3D self._get_hugepage_size() > + hugepages_total =3D self._get_hugepages_total() > + self._numa_nodes =3D self._get_numa_nodes() > + > + target_hugepages_total =3D int(hugepage_amount / hugepage_size) > + if hugepage_amount % hugepage_size: > + target_hugepages_total +=3D 1 > + if force_first_numa or hugepages_total !=3D target_hugepages_tot= al: > + # when forcing numa, we need to clear existing hugepages > regardless > + # of size, so they can be moved to the first numa node > + self._configure_huge_pages( > + target_hugepages_total, hugepage_size, force_first_numa > + ) > + else: > + self.logger.info("Hugepages already configured.") > + self._mount_huge_pages() > + > + def _get_hugepage_size(self) -> int: > + hugepage_size =3D self.remote_session.send_command( > + "awk '/Hugepagesize/ {print $2}' /proc/meminfo" > + ).stdout > + return int(hugepage_size) > + > + def _get_hugepages_total(self) -> int: > + hugepages_total =3D self.remote_session.send_command( > + "awk '/HugePages_Total/ { print $2 }' /proc/meminfo" > + ).stdout > + return int(hugepages_total) > + > + def _get_numa_nodes(self) -> list[int]: > + try: > + numa_range =3D self.remote_session.send_command( > + "cat /sys/devices/system/node/online", verify=3DTrue > + ).stdout > + numa_range =3D expand_range(numa_range) > + except RemoteCommandExecutionError: > + # the file doesn't exist, meaning the node doesn't support > numa > + numa_range =3D [] > + return numa_range > + > + def _mount_huge_pages(self) -> None: > + self.logger.info("Re-mounting Hugepages.") > + hugapge_fs_cmd =3D "awk '/hugetlbfs/ { print $2 }' /proc/mounts" > + self.remote_session.send_command(f"umount $({hugapge_fs_cmd})") > + result =3D self.remote_session.send_command(hugapge_fs_cmd) > + if result.stdout =3D=3D "": > + remote_mount_path =3D "/mnt/huge" > + self.remote_session.send_command(f"mkdir -p > {remote_mount_path}") > + self.remote_session.send_command( > + f"mount -t hugetlbfs nodev {remote_mount_path}" > + ) > + > + def _supports_numa(self) -> bool: > + # the system supports numa if self._numa_nodes is non-empty and > there are more > + # than one numa node (in the latter case it may actually support > numa, but > + # there's no reason to do any numa specific configuration) > + return len(self._numa_nodes) > 1 > + > + def _configure_huge_pages( > + self, amount: int, size: int, force_first_numa: bool > + ) -> None: > + self.logger.info("Configuring Hugepages.") > + hugepage_config_path =3D ( > + f"/sys/kernel/mm/hugepages/hugepages-{size}kB/nr_hugepages" > + ) > + if force_first_numa and self._supports_numa(): > + # clear non-numa hugepages > + self.remote_session.send_command( > + f"echo 0 | sudo tee {hugepage_config_path}" > + ) > + hugepage_config_path =3D ( > + > f"/sys/devices/system/node/node{self._numa_nodes[0]}/hugepages" > + f"/hugepages-{size}kB/nr_hugepages" > + ) > + > + self.remote_session.send_command( > + f"echo {amount} | sudo tee {hugepage_config_path}" > + ) > diff --git a/dts/framework/remote_session/os/os_session.py > b/dts/framework/remote_session/os/os_session.py > index 6f6b6a979e..f84f3ce63c 100644 > --- a/dts/framework/remote_session/os/os_session.py > +++ b/dts/framework/remote_session/os/os_session.py > @@ -144,3 +144,13 @@ def kill_cleanup_dpdk_apps(self, dpdk_prefix_list: > Iterable[str]) -> None: > Kill and cleanup all DPDK apps identified by dpdk_prefix_list. I= f > dpdk_prefix_list is empty, attempt to find running DPDK apps to > kill and clean. > """ > + > + @abstractmethod > + def setup_hugepages( > + self, hugepage_amount: int =3D -1, force_first_numa: bool =3D Fa= lse > + ) -> None: > + """ > + Get the node's Hugepage Size, configure the specified amount of > hugepages > + if needed and mount the hugepages if needed. > + If force_first_numa is True, configure hugepages just on the > first socket. > + """ > diff --git a/dts/framework/testbed_model/node/node.py > b/dts/framework/testbed_model/node/node.py > index 5ee7023335..96a1724f4c 100644 > --- a/dts/framework/testbed_model/node/node.py > +++ b/dts/framework/testbed_model/node/node.py > @@ -16,7 +16,7 @@ > ) > from framework.exception import NodeCleanupError, NodeSetupError, > convert_exception > from framework.logger import DTSLOG, getLogger > -from framework.remote_session import OSSession, create_session > +from framework.remote_session import Arch, OSSession, create_arch, > create_session > from framework.testbed_model.hw import CPUAmount, cpu_filter > > > @@ -33,6 +33,7 @@ class Node(object): > config: NodeConfiguration > cpus: list[CPU] > _other_sessions: list[OSSession] > + _arch: Arch > > def __init__(self, node_config: NodeConfiguration): > self.config =3D node_config > @@ -42,6 +43,7 @@ def __init__(self, node_config: NodeConfiguration): > self.logger =3D getLogger(self.name) > self.logger.info(f"Created node: {self.name}") > self.main_session =3D create_session(self.config, self.name, > self.logger) > + self._arch =3D create_arch(self.config) > self._get_remote_cpus() > > @convert_exception(NodeSetupError) > @@ -50,6 +52,7 @@ def setup_execution(self, execution_config: > ExecutionConfiguration) -> None: > Perform the execution setup that will be done for each execution > this node is part of. > """ > + self._setup_hugepages() > self._setup_execution(execution_config) > > def _setup_execution(self, execution_config: ExecutionConfiguration) > -> None: > @@ -145,6 +148,16 @@ def _get_remote_cpus(self) -> None: > self.logger.info("Getting CPU information.") > self.cpus =3D > self.main_session.get_remote_cpus(self.config.bypass_core0) > > + def _setup_hugepages(self): > + """ > + Setup hugepages on the Node. Different architectures can supply > different > + amounts of memory for hugepages and numa-based hugepage > allocation may need > + to be considered. > + """ > + self.main_session.setup_hugepages( > + self._arch.default_hugepage_memory, > self._arch.hugepage_force_first_numa > + ) > + > def close(self) -> None: > """ > Close all connections and free other resources. > -- > 2.30.2 > > --0000000000003e28fd05ed96ba33 Content-Type: text/html; charset="UTF-8" Content-Transfer-Encoding: quoted-printable


=
On Mon, Nov 14, 2022 at 11:54 AM Jura= j Linke=C5=A1 <juraj.linkes@pantheon.tech> wrote:
Setup hugepages on nodes. This is u= seful not only on SUT nodes, but
also on TG nodes which use TGs that utilize hugepages.

Signed-off-by: Juraj Linke=C5=A1 <juraj.linkes@pantheon.tech>
---
=C2=A0dts/framework/remote_session/__init__.py=C2=A0 =C2=A0 =C2=A0 |=C2=A0 = 1 +
=C2=A0dts/framework/remote_session/arch/__init__.py | 20 +++++
=C2=A0dts/framework/remote_session/arch/arch.py=C2=A0 =C2=A0 =C2=A0| 57 +++= ++++++++++
=C2=A0.../remote_session/os/linux_session.py=C2=A0 =C2=A0 =C2=A0 =C2=A0 | 8= 5 +++++++++++++++++++
=C2=A0dts/framework/remote_session/os/os_session.py | 10 +++
=C2=A0dts/framework/testbed_model/node/node.py=C2=A0 =C2=A0 =C2=A0 | 15 +++= -
=C2=A06 files changed, 187 insertions(+), 1 deletion(-)
=C2=A0create mode 100644 dts/framework/remote_session/arch/__init__.py
=C2=A0create mode 100644 dts/framework/remote_session/arch/arch.py

diff --git a/dts/framework/remote_session/__init__.py b/dts/framework/remot= e_session/__init__.py
index f2339b20bd..f0deeadac6 100644
--- a/dts/framework/remote_session/__init__.py
+++ b/dts/framework/remote_session/__init__.py
@@ -11,4 +11,5 @@

=C2=A0# pylama:ignore=3DW0611

+from .arch import Arch, create_arch
=C2=A0from .os import OSSession, create_session
diff --git a/dts/framework/remote_session/arch/__init__.py b/dts/framework/= remote_session/arch/__init__.py
new file mode 100644
index 0000000000..d78ad42ac5
--- /dev/null
+++ b/dts/framework/remote_session/arch/__init__.py
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2022 PANTHEON.tech s.r.o.
+
+from framework.config import Architecture, NodeConfiguration
+
+from .arch import PPC64, Arch, Arm64, i686, x86_32, x86_64
+
+
+def create_arch(node_config: NodeConfiguration) -> Arch:
+=C2=A0 =C2=A0 match node_config.arch:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 case Architecture.x86_64:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return x86_64()
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 case Architecture.x86_32:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return x86_32()
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 case Architecture.i686:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return i686()
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 case Architecture.ppc64le:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return PPC64()
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 case Architecture.arm64:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return Arm64()
diff --git a/dts/framework/remote_session/arch/arch.py b/dts/framework/remo= te_session/arch/arch.py
new file mode 100644
index 0000000000..05c7602def
--- /dev/null
+++ b/dts/framework/remote_session/arch/arch.py
@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2022 PANTHEON.tech s.r.o.
+
+
+class Arch(object):
+=C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 Stores architecture-specific information.
+=C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def default_hugepage_memory(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Return the default amount of memory allocated = for hugepages DPDK will use.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 The default is an amount equal to 256 2MB huge= pages (512MB memory).
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return 256 * 2048
+
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def hugepage_force_first_numa(self) -> bool:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 An architecture may need to force configuratio= n of hugepages to first socket.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return False
+
+
+class x86_64(Arch):
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def default_hugepage_memory(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return 4096 * 2048
+
+
+class x86_32(Arch):
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def hugepage_force_first_numa(self) -> bool:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return True
+
+
+class i686(Arch):
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def default_hugepage_memory(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return 512 * 2048
+
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def hugepage_force_first_numa(self) -> bool:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return True
+
+
+class PPC64(Arch):
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def default_hugepage_memory(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return 512 * 2048
+
+
+class Arm64(Arch):
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def default_hugepage_memory(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return 2048 * 2048
diff --git a/dts/framework/remote_session/os/linux_session.py b/dts/framewo= rk/remote_session/os/linux_session.py
index 21f117b714..fad33d7613 100644
--- a/dts/framework/remote_session/os/linux_session.py
+++ b/dts/framework/remote_session/os/linux_session.py
@@ -3,6 +3,8 @@
=C2=A0# Copyright(c) 2022 University of New Hampshire

=C2=A0from framework.config import CPU
+from framework.exception import RemoteCommandExecutionError
+from framework.utils import expand_range

=C2=A0from .posix_session import PosixSession

@@ -24,3 +26,86 @@ def get_remote_cpus(self, bypass_core0: bool) -> list= [CPU]:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0continue
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0cpus.append(CPU(int(cpu), i= nt(core), int(socket), int(node)))
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0return cpus
+
+=C2=A0 =C2=A0 def setup_hugepages(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, hugepage_amount: int =3D -1, force_first= _numa: bool =3D False

I think that huge= page_amount: int | None =3D None is better, since it expresses it is an opt= ional argument and the type checker will force anyone using the value to ch= eck if it is none, whereas that will not happen with -1.=C2=A0
=C2=A0
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info("Getting Hugepage info= rmation.")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 hugepage_size =3D self._get_hugepage_size() +=C2=A0 =C2=A0 =C2=A0 =C2=A0 hugepages_total =3D self._get_hugepages_total(= )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._numa_nodes =3D self._get_numa_nodes() +
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 target_hugepages_total =3D int(hugepage_amount= / hugepage_size)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if hugepage_amount % hugepage_size:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 target_hugepages_total +=3D 1 +=C2=A0 =C2=A0 =C2=A0 =C2=A0 if force_first_numa or hugepages_total !=3D ta= rget_hugepages_total:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # when forcing numa, we need to = clear existing hugepages regardless
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # of size, so they can be moved = to the first numa node
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._configure_huge_pages(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 target_hugepages_t= otal, hugepage_size, force_first_numa
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 else:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info("Hugepag= es already configured.")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._mount_huge_pages()
+
+=C2=A0 =C2=A0 def _get_hugepage_size(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 hugepage_size =3D self.remote_session.send_com= mand(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 "awk '/Hugepagesize/ {p= rint $2}' /proc/meminfo"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 ).stdout
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return int(hugepage_size)
+
+=C2=A0 =C2=A0 def _get_hugepages_total(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 hugepages_total =3D self.remote_session.send_c= ommand(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 "awk '/HugePages_Total/= { print $2 }' /proc/meminfo"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 ).stdout
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return int(hugepages_total)
+
+=C2=A0 =C2=A0 def _get_numa_nodes(self) -> list[int]:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 try:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 numa_range =3D self.remote_sessi= on.send_command(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 "cat /sys/dev= ices/system/node/online", verify=3DTrue=C2=A0
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ).stdout
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 numa_range =3D expand_range(numa= _range)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 except RemoteCommandExecutionError:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # the file doesn't exist, me= aning the node doesn't support numa
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 numa_range =3D []
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return numa_range
+
+=C2=A0 =C2=A0 def _mount_huge_pages(self) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info("Re-mounting Hugepages= .")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 hugapge_fs_cmd =3D "awk '/hugetlbfs/ = { print $2 }' /proc/mounts"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command(f"umount= $({hugapge_fs_cmd})")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 result =3D self.remote_session.send_command(hu= gapge_fs_cmd)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if result.stdout =3D=3D "":
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 remote_mount_path =3D "/mnt= /huge"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command= (f"mkdir -p {remote_mount_path}")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command= (
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"mount -t hu= getlbfs nodev {remote_mount_path}"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+
+=C2=A0 =C2=A0 def _supports_numa(self) -> bool:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # the system supports numa if self._numa_nodes= is non-empty and there are more
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # than one numa node (in the latter case it ma= y actually support numa, but
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # there's no reason to do any numa specifi= c configuration)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return len(self._numa_nodes) > 1
+
+=C2=A0 =C2=A0 def _configure_huge_pages(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, amount: int, size: int, force_first_numa= : bool
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info("Configuring Hugepages= .")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 hugepage_config_path =3D (
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"/sys/kernel/mm/hugepages/= hugepages-{size}kB/nr_hugepages"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if force_first_numa and self._supports_numa():=
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # clear non-numa hugepages
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command= (
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"echo 0 | su= do tee {hugepage_config_path}"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 hugepage_config_path =3D (
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"/sys/device= s/system/node/node{self._numa_nodes[0]}/hugepages"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"/hugepages-= {size}kB/nr_hugepages"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"echo {amount} | sudo tee = {hugepage_config_path}"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
diff --git a/dts/framework/remote_session/os/os_session.py b/dts/framework/= remote_session/os/os_session.py
index 6f6b6a979e..f84f3ce63c 100644
--- a/dts/framework/remote_session/os/os_session.py
+++ b/dts/framework/remote_session/os/os_session.py
@@ -144,3 +144,13 @@ def kill_cleanup_dpdk_apps(self, dpdk_prefix_list: Ite= rable[str]) -> None:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0Kill and cleanup all DPDK apps identified= by dpdk_prefix_list. If
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0dpdk_prefix_list is empty, attempt to fin= d running DPDK apps to kill and clean.
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""
+
+=C2=A0 =C2=A0 @abstractmethod
+=C2=A0 =C2=A0 def setup_hugepages(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, hugepage_amount: int =3D -1, force_first= _numa: bool =3D False
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Get the node's Hugepage Size, configure th= e specified amount of hugepages
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if needed and mount the hugepages if needed. +=C2=A0 =C2=A0 =C2=A0 =C2=A0 If force_first_numa is True, configure hugepag= es just on the first socket.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
diff --git a/dts/framework/testbed_model/node/node.py b/dts/framework/testb= ed_model/node/node.py
index 5ee7023335..96a1724f4c 100644
--- a/dts/framework/testbed_model/node/node.py
+++ b/dts/framework/testbed_model/node/node.py
@@ -16,7 +16,7 @@
=C2=A0)
=C2=A0from framework.exception import NodeCleanupError, NodeSetupError, con= vert_exception
=C2=A0from framework.logger import DTSLOG, getLogger
-from framework.remote_session import OSSession, create_session
+from framework.remote_session import Arch, OSSession, create_arch, create_= session
=C2=A0from framework.testbed_model.hw import CPUAmount, cpu_filter


@@ -33,6 +33,7 @@ class Node(object):
=C2=A0 =C2=A0 =C2=A0config: NodeConfiguration
=C2=A0 =C2=A0 =C2=A0cpus: list[CPU]
=C2=A0 =C2=A0 =C2=A0_other_sessions: list[OSSession]
+=C2=A0 =C2=A0 _arch: Arch

=C2=A0 =C2=A0 =C2=A0def __init__(self, node_config: NodeConfiguration):
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.config =3D node_config
@@ -42,6 +43,7 @@ def __init__(self, node_config: NodeConfiguration):
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.logger =3D getLogger(self.name)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.logger.info(f"Created node:= {self.na= me}")
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.main_session =3D create_session(self= .config, = self.name, self.logger)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._arch =3D create_arch(self.config)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self._get_remote_cpus()

=C2=A0 =C2=A0 =C2=A0@convert_exception(NodeSetupError)
@@ -50,6 +52,7 @@ def setup_execution(self, execution_config: ExecutionConf= iguration) -> None:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0Perform the execution setup that will be = done for each execution
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0this node is part of.
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._setup_hugepages()
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self._setup_execution(execution_config)
=C2=A0 =C2=A0 =C2=A0def _setup_execution(self, execution_config: ExecutionC= onfiguration) -> None:
@@ -145,6 +148,16 @@ def _get_remote_cpus(self) -> None:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.logger.info("Getting CPU in= formation.")
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.cpus =3D self.main_session.get_remot= e_cpus(self.config.bypass_core0)

+=C2=A0 =C2=A0 def _setup_hugepages(self):
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Setup hugepages on the Node. Different archite= ctures can supply different
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 amounts of memory for hugepages and numa-based= hugepage allocation may need
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 to be considered.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.main_session.setup_hugepages(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._arch.default_hugepage_memo= ry, self._arch.hugepage_force_first_numa
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+
=C2=A0 =C2=A0 =C2=A0def close(self) -> None:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0Close all connections and free other reso= urces.
--
2.30.2


On Mon, Nov 14, 2022 at 11:54 AM Juraj Linke=C5=A1 <j= uraj.linkes@pantheon.tech> wrote:
Setup hugepages on nodes. This is useful not only on S= UT nodes, but
also on TG nodes which use TGs that utilize hugepages.

Signed-off-by: Juraj Linke=C5=A1 <juraj.linkes@pantheon.tech>
---
=C2=A0dts/framework/remote_session/__init__.py=C2=A0 =C2=A0 =C2=A0 |=C2=A0 = 1 +
=C2=A0dts/framework/remote_session/arch/__init__.py | 20 +++++
=C2=A0dts/framework/remote_session/arch/arch.py=C2=A0 =C2=A0 =C2=A0| 57 +++= ++++++++++
=C2=A0.../remote_session/os/linux_session.py=C2=A0 =C2=A0 =C2=A0 =C2=A0 | 8= 5 +++++++++++++++++++
=C2=A0dts/framework/remote_session/os/os_session.py | 10 +++
=C2=A0dts/framework/testbed_model/node/node.py=C2=A0 =C2=A0 =C2=A0 | 15 +++= -
=C2=A06 files changed, 187 insertions(+), 1 deletion(-)
=C2=A0create mode 100644 dts/framework/remote_session/arch/__init__.py
=C2=A0create mode 100644 dts/framework/remote_session/arch/arch.py

diff --git a/dts/framework/remote_session/__init__.py b/dts/framework/remot= e_session/__init__.py
index f2339b20bd..f0deeadac6 100644
--- a/dts/framework/remote_session/__init__.py
+++ b/dts/framework/remote_session/__init__.py
@@ -11,4 +11,5 @@

=C2=A0# pylama:ignore=3DW0611

+from .arch import Arch, create_arch
=C2=A0from .os import OSSession, create_session
diff --git a/dts/framework/remote_session/arch/__init__.py b/dts/framework/= remote_session/arch/__init__.py
new file mode 100644
index 0000000000..d78ad42ac5
--- /dev/null
+++ b/dts/framework/remote_session/arch/__init__.py
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2022 PANTHEON.tech s.r.o.
+
+from framework.config import Architecture, NodeConfiguration
+
+from .arch import PPC64, Arch, Arm64, i686, x86_32, x86_64
+
+
+def create_arch(node_config: NodeConfiguration) -> Arch:
+=C2=A0 =C2=A0 match node_config.arch:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 case Architecture.x86_64:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return x86_64()
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 case Architecture.x86_32:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return x86_32()
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 case Architecture.i686:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return i686()
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 case Architecture.ppc64le:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return PPC64()
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 case Architecture.arm64:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return Arm64()
diff --git a/dts/framework/remote_session/arch/arch.py b/dts/framework/remo= te_session/arch/arch.py
new file mode 100644
index 0000000000..05c7602def
--- /dev/null
+++ b/dts/framework/remote_session/arch/arch.py
@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2022 PANTHEON.tech s.r.o.
+
+
+class Arch(object):
+=C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 Stores architecture-specific information.
+=C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def default_hugepage_memory(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Return the default amount of memory allocated = for hugepages DPDK will use.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 The default is an amount equal to 256 2MB huge= pages (512MB memory).
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return 256 * 2048
+
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def hugepage_force_first_numa(self) -> bool:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 An architecture may need to force configuratio= n of hugepages to first socket.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return False
+
+
+class x86_64(Arch):
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def default_hugepage_memory(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return 4096 * 2048
+
+
+class x86_32(Arch):
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def hugepage_force_first_numa(self) -> bool:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return True
+
+
+class i686(Arch):
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def default_hugepage_memory(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return 512 * 2048
+
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def hugepage_force_first_numa(self) -> bool:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return True
+
+
+class PPC64(Arch):
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def default_hugepage_memory(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return 512 * 2048
+
+
+class Arm64(Arch):
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def default_hugepage_memory(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return 2048 * 2048
diff --git a/dts/framework/remote_session/os/linux_session.py b/dts/framewo= rk/remote_session/os/linux_session.py
index 21f117b714..fad33d7613 100644
--- a/dts/framework/remote_session/os/linux_session.py
+++ b/dts/framework/remote_session/os/linux_session.py
@@ -3,6 +3,8 @@
=C2=A0# Copyright(c) 2022 University of New Hampshire

=C2=A0from framework.config import CPU
+from framework.exception import RemoteCommandExecutionError
+from framework.utils import expand_range

=C2=A0from .posix_session import PosixSession

@@ -24,3 +26,86 @@ def get_remote_cpus(self, bypass_core0: bool) -> list= [CPU]:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0continue
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0cpus.append(CPU(int(cpu), i= nt(core), int(socket), int(node)))
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0return cpus
+
+=C2=A0 =C2=A0 def setup_hugepages(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, hugepage_amount: int =3D -1, force_first= _numa: bool =3D False
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info("Getting Hugepage info= rmation.")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 hugepage_size =3D self._get_hugepage_size() +=C2=A0 =C2=A0 =C2=A0 =C2=A0 hugepages_total =3D self._get_hugepages_total(= )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._numa_nodes =3D self._get_numa_nodes() +
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 target_hugepages_total =3D int(hugepage_amount= / hugepage_size)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if hugepage_amount % hugepage_size:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 target_hugepages_total +=3D 1 +=C2=A0 =C2=A0 =C2=A0 =C2=A0 if force_first_numa or hugepages_total !=3D ta= rget_hugepages_total:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # when forcing numa, we need to = clear existing hugepages regardless
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # of size, so they can be moved = to the first numa node
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._configure_huge_pages(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 target_hugepages_t= otal, hugepage_size, force_first_numa
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 else:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info("Hugepag= es already configured.")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._mount_huge_pages()
+
+=C2=A0 =C2=A0 def _get_hugepage_size(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 hugepage_size =3D self.remote_session.send_com= mand(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 "awk '/Hugepagesize/ {p= rint $2}' /proc/meminfo"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 ).stdout
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return int(hugepage_size)
+
+=C2=A0 =C2=A0 def _get_hugepages_total(self) -> int:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 hugepages_total =3D self.remote_session.send_c= ommand(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 "awk '/HugePages_Total/= { print $2 }' /proc/meminfo"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 ).stdout
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return int(hugepages_total)
+
+=C2=A0 =C2=A0 def _get_numa_nodes(self) -> list[int]:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 try:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 numa_range =3D self.remote_sessi= on.send_command(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 "cat /sys/dev= ices/system/node/online", verify=3DTrue
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ).stdout
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 numa_range =3D expand_range(numa= _range)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 except RemoteCommandExecutionError:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # the file doesn't exist, me= aning the node doesn't support numa
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 numa_range =3D []
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return numa_range
+
+=C2=A0 =C2=A0 def _mount_huge_pages(self) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info("Re-mounting Hugepages= .")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 hugapge_fs_cmd =3D "awk '/hugetlbfs/ = { print $2 }' /proc/mounts"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command(f"umount= $({hugapge_fs_cmd})")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 result =3D self.remote_session.send_command(hu= gapge_fs_cmd)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if result.stdout =3D=3D "":
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 remote_mount_path =3D "/mnt= /huge"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command= (f"mkdir -p {remote_mount_path}")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command= (
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"mount -t hu= getlbfs nodev {remote_mount_path}"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+
+=C2=A0 =C2=A0 def _supports_numa(self) -> bool:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # the system supports numa if self._numa_nodes= is non-empty and there are more
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # than one numa node (in the latter case it ma= y actually support numa, but
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # there's no reason to do any numa specifi= c configuration)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return len(self._numa_nodes) > 1
+
+=C2=A0 =C2=A0 def _configure_huge_pages(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, amount: int, size: int, force_first_numa= : bool
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info("Configuring Hugepages= .")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 hugepage_config_path =3D (
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"/sys/kernel/mm/hugepages/= hugepages-{size}kB/nr_hugepages"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if force_first_numa and self._supports_numa():=
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # clear non-numa hugepages
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command= (
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"echo 0 | su= do tee {hugepage_config_path}"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 hugepage_config_path =3D (
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"/sys/device= s/system/node/node{self._numa_nodes[0]}/hugepages"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"/hugepages-= {size}kB/nr_hugepages"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"echo {amount} | sudo tee = {hugepage_config_path}"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
diff --git a/dts/framework/remote_session/os/os_session.py b/dts/framework/= remote_session/os/os_session.py
index 6f6b6a979e..f84f3ce63c 100644
--- a/dts/framework/remote_session/os/os_session.py
+++ b/dts/framework/remote_session/os/os_session.py
@@ -144,3 +144,13 @@ def kill_cleanup_dpdk_apps(self, dpdk_prefix_list: Ite= rable[str]) -> None:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0Kill and cleanup all DPDK apps identified= by dpdk_prefix_list. If
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0dpdk_prefix_list is empty, attempt to fin= d running DPDK apps to kill and clean.
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""
+
+=C2=A0 =C2=A0 @abstractmethod
+=C2=A0 =C2=A0 def setup_hugepages(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, hugepage_amount: int =3D -1, force_first= _numa: bool =3D False
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Get the node's Hugepage Size, configure th= e specified amount of hugepages
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if needed and mount the hugepages if needed. +=C2=A0 =C2=A0 =C2=A0 =C2=A0 If force_first_numa is True, configure hugepag= es just on the first socket.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
diff --git a/dts/framework/testbed_model/node/node.py b/dts/framework/testb= ed_model/node/node.py
index 5ee7023335..96a1724f4c 100644
--- a/dts/framework/testbed_model/node/node.py
+++ b/dts/framework/testbed_model/node/node.py
@@ -16,7 +16,7 @@
=C2=A0)
=C2=A0from framework.exception import NodeCleanupError, NodeSetupError, con= vert_exception
=C2=A0from framework.logger import DTSLOG, getLogger
-from framework.remote_session import OSSession, create_session
+from framework.remote_session import Arch, OSSession, create_arch, create_= session
=C2=A0from framework.testbed_model.hw import CPUAmount, cpu_filter


@@ -33,6 +33,7 @@ class Node(object):
=C2=A0 =C2=A0 =C2=A0config: NodeConfiguration
=C2=A0 =C2=A0 =C2=A0cpus: list[CPU]
=C2=A0 =C2=A0 =C2=A0_other_sessions: list[OSSession]
+=C2=A0 =C2=A0 _arch: Arch

=C2=A0 =C2=A0 =C2=A0def __init__(self, node_config: NodeConfiguration):
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.config =3D node_config
@@ -42,6 +43,7 @@ def __init__(self, node_config: NodeConfiguration):
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.logger =3D getLogger(self.name)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.logger.info(f"Created node:= {self.na= me}")
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.main_session =3D create_session(self= .config, = self.name, self.logger)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._arch =3D create_arch(self.config)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self._get_remote_cpus()

=C2=A0 =C2=A0 =C2=A0@convert_exception(NodeSetupError)
@@ -50,6 +52,7 @@ def setup_execution(self, execution_config: ExecutionConf= iguration) -> None:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0Perform the execution setup that will be = done for each execution
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0this node is part of.
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._setup_hugepages()
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self._setup_execution(execution_config)
=C2=A0 =C2=A0 =C2=A0def _setup_execution(self, execution_config: ExecutionC= onfiguration) -> None:
@@ -145,6 +148,16 @@ def _get_remote_cpus(self) -> None:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.logger.info("Getting CPU in= formation.")
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.cpus =3D self.main_session.get_remot= e_cpus(self.config.bypass_core0)

+=C2=A0 =C2=A0 def _setup_hugepages(self):
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Setup hugepages on the Node. Different archite= ctures can supply different
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 amounts of memory for hugepages and numa-based= hugepage allocation may need
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 to be considered.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.main_session.setup_hugepages(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._arch.default_hugepage_memo= ry, self._arch.hugepage_force_first_numa
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+
=C2=A0 =C2=A0 =C2=A0def close(self) -> None:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0Close all connections and free other reso= urces.
--
2.30.2

--0000000000003e28fd05ed96ba33--