From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 7D54C45823; Tue, 20 Aug 2024 17:35:37 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 5506340DCA; Tue, 20 Aug 2024 17:35:27 +0200 (CEST) Received: from mgamail.intel.com (mgamail.intel.com [192.198.163.7]) by mails.dpdk.org (Postfix) with ESMTP id 42E7940B9A for ; Tue, 20 Aug 2024 17:35:25 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1724168125; x=1755704125; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=PoUwzpzUQrJucspY/s65sHdTts4k/IkzdSE4QeeFQ24=; b=cd6FLpTG24oJCvJNc+IH0rUb6tIxl+HTJNB4lTos0gXIGig1EJ4GTkAO BVpeQkM3emw3c9jsg+7x/LeCajHaXHcrqLMq3f0S73sdqsoInlGwiOsFV u4Km+IFl784OW+WG2zmEV2VhxRnytomo1fmDH2m5sdZYjbN5yU/oZQWZ4 8jKQ5YnujyLU1c0nGoy8pZowNtV3v6zjFyyZSDYKrRMYX7oSKejWFQipc vW4eITzy+oKfZjNzFHiEbXMEDL5Fz3c9m6dyTb6bOGb5CIhLfrHFBiVHa In1wcTvFgvHKfOmPuDndweHgLlH9opg62Qu0jaRE4i9Vl19UKCHYuUylt w==; X-CSE-ConnectionGUID: OWZ/Av5KTECgMc8hS4ZdcA== X-CSE-MsgGUID: D1WFSEXRTLes9+AEf05RQg== X-IronPort-AV: E=McAfee;i="6700,10204,11170"; a="47872782" X-IronPort-AV: E=Sophos;i="6.10,162,1719903600"; d="scan'208";a="47872782" Received: from orviesa006.jf.intel.com ([10.64.159.146]) by fmvoesa101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 20 Aug 2024 08:35:24 -0700 X-CSE-ConnectionGUID: Kmo9oqz5TouhBXUZvfcecQ== X-CSE-MsgGUID: uLHYQAZCRVSOn0gnEk2XwQ== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="6.10,162,1719903600"; d="scan'208";a="61074483" Received: from silpixa00401119.ir.intel.com ([10.55.129.167]) by orviesa006.jf.intel.com with ESMTP; 20 Aug 2024 08:35:23 -0700 From: Anatoly Burakov To: dev@dpdk.org, Robin Jarry Cc: bruce.richardson@intel.com Subject: [PATCH v3 3/4] usertools/dpdk-hugepages.py: update coding style Date: Tue, 20 Aug 2024 16:35:16 +0100 Message-ID: <1c9f589c434cfb91c53dae7528cfc165c976b0c6.1724168117.git.anatoly.burakov@intel.com> X-Mailer: git-send-email 2.43.5 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Update coding style: - Make the code PEP-484 compliant - Add more comments, improve readability, use f-strings everywhere - Use quotes consistently - Address all Python static analysis (e.g. mypy, pylint) warnings - Improve error handling - Refactor printing and sysfs/procfs access functions - Sort output by NUMA node Signed-off-by: Anatoly Burakov --- Notes: v1 -> v2: - Added commit that sorted output by NUMA node v2 -> v3: - Rewrite of the script as suggested by reviewers usertools/dpdk-hugepages.py | 456 +++++++++++++++++++++--------------- 1 file changed, 273 insertions(+), 183 deletions(-) diff --git a/usertools/dpdk-hugepages.py b/usertools/dpdk-hugepages.py index bf2575ba36..510822af60 100755 --- a/usertools/dpdk-hugepages.py +++ b/usertools/dpdk-hugepages.py @@ -1,167 +1,136 @@ #! /usr/bin/env python3 # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2020 Microsoft Corporation -"""Script to query and setup huge pages for DPDK applications.""" + +'''Script to query and setup huge pages for DPDK applications.''' import argparse -import glob import os import re +import subprocess import sys +import typing as T from math import log2 # Standard binary prefix -BINARY_PREFIX = "KMG" +BINARY_PREFIX = 'KMG' # systemd mount point for huge pages -HUGE_MOUNT = "/dev/hugepages" +HUGE_MOUNT = '/dev/hugepages' +# default directory for non-NUMA huge pages +NO_NUMA_HUGE_DIR = '/sys/kernel/mm/hugepages' +# default base directory for NUMA nodes +NUMA_NODE_BASE_DIR = '/sys/devices/system/node' +# procfs paths +MEMINFO_PATH = '/proc/meminfo' +MOUNTS_PATH = '/proc/mounts' -def fmt_memsize(kb): +class HugepageMount: + '''Mount operations for huge page filesystem.''' + + def __init__(self, path: str, mounted: bool): + self.path = path + # current mount status + self.mounted = mounted + + def mount(self, pagesize_kb: int, + user: T.Optional[str], group: T.Optional[str]) -> None: + '''Mount the huge TLB file system''' + if self.mounted: + return + cmd = ['mount', '-t', 'hugetlbfs'] + cmd += ['-o', f'pagesize={pagesize_kb * 1024}'] + if user is not None: + cmd += ['-o', f'uid={user}'] + if group is not None: + cmd += ['-o', f'gid={group}'] + cmd += ['nodev', self.path] + + subprocess.run(cmd, check=True) + self.mounted = True + + def unmount(self) -> None: + '''Unmount the huge TLB file system (if mounted)''' + if self.mounted: + subprocess.run(['umount', self.path], check=True) + self.mounted = False + + +class HugepageRes: + '''Huge page reserve operations. Can be NUMA-node-specific.''' + + def __init__(self, path: str, node: T.Optional[int] = None): + self.path = path + # if this is a per-NUMA node huge page dir, store the node number + self.node = node + self.valid_page_sizes = self._get_valid_page_sizes() + + def _get_valid_page_sizes(self) -> T.List[int]: + '''Extract valid huge page sizes''' + return [get_memsize(d.split('-')[1]) + for d in os.listdir(self.path)] + + def _nr_pages_path(self, sz: int) -> str: + if sz not in self.valid_page_sizes: + raise ValueError(f'Invalid page size {sz}. ' + f'Valid sizes: {self.valid_page_sizes}') + return os.path.join(self.path, f'hugepages-{sz}kB', 'nr_hugepages') + + def __getitem__(self, sz: int) -> int: + '''Get current number of reserved pages of specified size''' + with open(self._nr_pages_path(sz), encoding='utf-8') as f: + return int(f.read()) + + def __setitem__(self, sz: int, nr_pages: int) -> None: + '''Set number of reserved pages of specified size''' + with open(self._nr_pages_path(sz), 'w', encoding='utf-8') as f: + f.write(f'{nr_pages}\n') + + +def fmt_memsize(kb: int) -> str: '''Format memory size in kB into conventional format''' logk = int(log2(kb) / 10) suffix = BINARY_PREFIX[logk] unit = 2**(logk * 10) - return '{}{}b'.format(int(kb / unit), suffix) + return f'{int(kb / unit)}{suffix}b' -def get_memsize(arg): +def get_memsize(arg: str) -> int: '''Convert memory size with suffix to kB''' - match = re.match(r'(\d+)([' + BINARY_PREFIX + r']?)$', arg.upper()) + # arg may have a 'b' at the end + if arg[-1].lower() == 'b': + arg = arg[:-1] + match = re.match(rf'(\d+)([{BINARY_PREFIX}]?)$', arg.upper()) if match is None: - sys.exit('{} is not a valid size'.format(arg)) + raise ValueError(f'{arg} is not a valid size') num = float(match.group(1)) suffix = match.group(2) - if suffix == "": + if not suffix: return int(num / 1024) idx = BINARY_PREFIX.find(suffix) return int(num * (2**(idx * 10))) -def is_numa(): - '''Test if NUMA is necessary on this system''' - return os.path.exists('/sys/devices/system/node') +def is_numa() -> bool: + '''Check if NUMA is supported''' + return os.path.exists(NUMA_NODE_BASE_DIR) -def get_valid_page_sizes(path): - '''Extract valid hugepage sizes''' - dir = os.path.dirname(path) - pg_sizes = (d.split("-")[1] for d in os.listdir(dir)) - return " ".join(pg_sizes) - - -def get_hugepages(path): - '''Read number of reserved pages''' - with open(path + '/nr_hugepages') as nr_hugepages: - return int(nr_hugepages.read()) - return 0 - - -def set_hugepages(path, reqpages): - '''Write the number of reserved huge pages''' - filename = path + '/nr_hugepages' - try: - with open(filename, 'w') as nr_hugepages: - nr_hugepages.write('{}\n'.format(reqpages)) - except PermissionError: - sys.exit('Permission denied: need to be root!') - except FileNotFoundError: - sys.exit("Invalid page size. Valid page sizes: {}".format( - get_valid_page_sizes(path))) - gotpages = get_hugepages(path) - if gotpages != reqpages: - sys.exit('Unable to set pages ({} instead of {} in {}).'.format( - gotpages, reqpages, filename)) - - -def show_numa_pages(): - '''Show huge page reservations on Numa system''' - print('Node Pages Size Total') - for numa_path in glob.glob('/sys/devices/system/node/node*'): - node = numa_path[29:] # slice after /sys/devices/system/node/node - path = numa_path + '/hugepages' - if not os.path.exists(path): - continue - for hdir in os.listdir(path): - pages = get_hugepages(path + '/' + hdir) - if pages > 0: - kb = int(hdir[10:-2]) # slice out of hugepages-NNNkB - print('{:<4} {:<5} {:<6} {}'.format(node, pages, - fmt_memsize(kb), - fmt_memsize(pages * kb))) - - -def show_non_numa_pages(): - '''Show huge page reservations on non Numa system''' - print('Pages Size Total') - path = '/sys/kernel/mm/hugepages' - for hdir in os.listdir(path): - pages = get_hugepages(path + '/' + hdir) - if pages > 0: - kb = int(hdir[10:-2]) - print('{:<5} {:<6} {}'.format(pages, fmt_memsize(kb), - fmt_memsize(pages * kb))) - - -def show_pages(): - '''Show existing huge page settings''' - if is_numa(): - show_numa_pages() - else: - show_non_numa_pages() - - -def clear_pages(): - '''Clear all existing huge page mappings''' - if is_numa(): - dirs = glob.glob( - '/sys/devices/system/node/node*/hugepages/hugepages-*') - else: - dirs = glob.glob('/sys/kernel/mm/hugepages/hugepages-*') - - for path in dirs: - set_hugepages(path, 0) - - -def default_pagesize(): +def default_pagesize() -> int: '''Get default huge page size from /proc/meminfo''' - with open('/proc/meminfo') as meminfo: + key = 'Hugepagesize' + with open(MEMINFO_PATH, encoding='utf-8') as meminfo: for line in meminfo: - if line.startswith('Hugepagesize:'): + if line.startswith(f'{key}:'): return int(line.split()[1]) - return None + raise KeyError(f'"{key}" not found in {MEMINFO_PATH}') -def set_numa_pages(pages, hugepgsz, node=None): - '''Set huge page reservation on Numa system''' - if node: - nodes = ['/sys/devices/system/node/node{}/hugepages'.format(node)] - else: - nodes = glob.glob('/sys/devices/system/node/node*/hugepages') - - for node_path in nodes: - huge_path = '{}/hugepages-{}kB'.format(node_path, hugepgsz) - set_hugepages(huge_path, pages) - - -def set_non_numa_pages(pages, hugepgsz): - '''Set huge page reservation on non Numa system''' - path = '/sys/kernel/mm/hugepages/hugepages-{}kB'.format(hugepgsz) - set_hugepages(path, pages) - - -def reserve_pages(pages, hugepgsz, node=None): - '''Set the number of huge pages to be reserved''' - if node or is_numa(): - set_numa_pages(pages, hugepgsz, node=node) - else: - set_non_numa_pages(pages, hugepgsz) - - -def get_mountpoints(): - '''Get list of where hugepage filesystem is mounted''' - mounted = [] - with open('/proc/mounts') as mounts: +def get_hugetlbfs_mountpoints() -> T.List[str]: + '''Get list of where huge page filesystem is mounted''' + mounted: T.List[str] = [] + with open(MOUNTS_PATH, encoding='utf-8') as mounts: for line in mounts: fields = line.split() if fields[2] != 'hugetlbfs': @@ -170,43 +139,144 @@ def get_mountpoints(): return mounted -def mount_huge(pagesize, mountpoint, user, group): - '''Mount the huge TLB file system''' - if mountpoint in get_mountpoints(): - print(mountpoint, "already mounted") - return - cmd = "mount -t hugetlbfs" - if pagesize: - cmd += ' -o pagesize={}'.format(pagesize * 1024) - if user: - cmd += ' -o uid=' + user - if group: - cmd += ' -o gid=' + group - cmd += ' nodev ' + mountpoint - os.system(cmd) +def print_row(cells: T.Tuple[str, ...], widths: T.List[int]) -> None: + '''Print a row of a table with the given column widths''' + first, *rest = cells + w_first, *w_rest = widths + first_end = ' ' * 2 + rest_end = ' ' * 2 + print(first.ljust(w_first), end=first_end) + for cell, width in zip(rest, w_rest): + print(cell.rjust(width), end=rest_end) + print() -def umount_huge(mountpoint): - '''Unmount the huge TLB file system (if mounted)''' - if mountpoint in get_mountpoints(): - os.system("umount " + mountpoint) +def print_hp_status(hp_res: T.List[HugepageRes]) -> None: + '''Display status of huge page reservations''' + numa = is_numa() -def show_mount(): - '''Show where huge page filesystem is mounted''' - mounted = get_mountpoints() - if mounted: - print("Hugepages mounted on", *mounted) + # print out huge page information in a table + rows: T.List[T.Tuple[str, ...]] + headers: T.Tuple[str, ...] + if numa: + headers = 'Node', 'Pages', 'Size', 'Total' + rows = [ + ( + str(hp.node), + str(nr_pages), + fmt_memsize(sz), + fmt_memsize(sz * nr_pages) + ) + # iterate over each huge page sysfs node... + for hp in hp_res + # ...and each page size within that node... + for sz in hp.valid_page_sizes + # ...we need number of pages multiple times, so we read it here... + for nr_pages in [hp[sz]] + # ...include this row only if there are pages reserved + if nr_pages + ] else: - print("Hugepages not mounted") + headers = 'Pages', 'Size', 'Total' + # if NUMA is disabled, we know there's only one huge page dir + hp = hp_res[0] + rows = [ + ( + str(nr_pages), + fmt_memsize(sz), + fmt_memsize(sz * nr_pages) + ) + # iterate over each page size within the huge page dir + for sz in hp.valid_page_sizes + # read number of pages for this size + for nr_pages in [hp[sz]] + # skip if no pages + if nr_pages + ] + if not rows: + print('No huge pages reserved') + return + + # find max widths for each column, including header and rows + col_widths = [ + max(len(tup[col_idx]) for tup in rows + [headers]) + for col_idx in range(len(headers)) + ] + + # print everything + print_row(headers, col_widths) + for r in rows: + print_row(r, col_widths) + + +def print_mount_status() -> None: + '''Display status of huge page filesystem mounts''' + mounted = get_hugetlbfs_mountpoints() + if not mounted: + print('No huge page filesystems mounted') + return + print('Huge page filesystems mounted at:', *mounted, sep=' ') + + +def scan_huge_dirs(node: T.Optional[int]) -> T.List[HugepageRes]: + '''Return a HugepageRes object for each huge page directory''' + # if NUMA is enabled, scan per-NUMA node huge pages + if is_numa(): + # helper function to extract node number from directory name + def _get_node(path: str) -> T.Optional[int]: + m = re.match(r'node(\d+)', os.path.basename(path)) + return int(m.group(1)) if m else None + # we want a sorted list of NUMA nodes + nodes = sorted(n + # iterate over all directories in the base directory + for d in os.listdir(NUMA_NODE_BASE_DIR) + # extract the node number from the directory name + for n in [_get_node(d)] + # filter out None values (non-NUMA node directories) + if n is not None) + return [ + HugepageRes(f'{NUMA_NODE_BASE_DIR}/node{n}/hugepages', n) + for n in nodes + # if user requested a specific node, only include that one + if node is None or n == node + ] + # otherwise, use non-NUMA huge page directory + if node is not None: + raise ValueError('NUMA node requested but not supported') + return [HugepageRes(NO_NUMA_HUGE_DIR)] + + +def try_reserve_huge_pages(hp_res: T.List[HugepageRes], mem_sz: str, + pagesize_kb: int) -> None: + '''Reserve huge pages if possible''' + reserve_kb = get_memsize(mem_sz) + + # is this a valid request? + if reserve_kb % pagesize_kb != 0: + fmt_res = fmt_memsize(reserve_kb) + fmt_sz = fmt_memsize(pagesize_kb) + raise ValueError(f'Huge reservation {fmt_res} is ' + f'not a multiple of page size {fmt_sz}') + + # request is valid, reserve pages + for hp in hp_res: + req = reserve_kb // pagesize_kb + hp[pagesize_kb] = req + got = hp[pagesize_kb] + # did we fulfill our request? + if got != req: + raise OSError(f'Failed to reserve {req} pages of size ' + f'{fmt_memsize(pagesize_kb)}, ' + f'got {got} pages instead') def main(): '''Process the command line arguments and setup huge pages''' parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, - description="Setup huge pages", - epilog=""" + description='Setup huge pages', + epilog=''' Examples: To display current huge page settings: @@ -214,94 +284,114 @@ def main(): To a complete setup of with 2 Gigabyte of 1G huge pages: %(prog)s -p 1G --setup 2G -""") +''') parser.add_argument( '--show', '-s', action='store_true', - help="print the current huge page configuration") + help='Print current huge page configuration') parser.add_argument( - '--clear', '-c', action='store_true', help="clear existing huge pages") + '--clear', '-c', action='store_true', help='Clear existing huge pages') parser.add_argument( '--mount', '-m', action='store_true', - help='mount the huge page filesystem') + help='Mount the huge page filesystem') parser.add_argument( '--unmount', '-u', action='store_true', - help='unmount the system huge page directory') + help='Unmount the system huge page directory') parser.add_argument( '--directory', '-d', metavar='DIR', default=HUGE_MOUNT, - help='mount point') + help='Mount point for huge pages') parser.add_argument( '--user', '-U', metavar='UID', - help='set the mounted directory owner user') + help='Set the mounted directory owner user') parser.add_argument( '--group', '-G', metavar='GID', - help='set the mounted directory owner group') + help='Set the mounted directory owner group') parser.add_argument( - '--node', '-n', help='select numa node to reserve pages on') + '--node', '-n', type=int, help='Select numa node to reserve pages on') parser.add_argument( '--pagesize', '-p', metavar='SIZE', - help='choose huge page size to use') + help='Choose huge page size to use') parser.add_argument( '--reserve', '-r', metavar='SIZE', - help='reserve huge pages. Size is in bytes with K, M, or G suffix') + help='Reserve huge pages. Size is in bytes with K, M, or G suffix') parser.add_argument( '--setup', metavar='SIZE', - help='setup huge pages by doing clear, unmount, reserve and mount') + help='Setup huge pages by doing clear, unmount, reserve and mount') args = parser.parse_args() + # setup is clear, then unmount, then reserve, then mount if args.setup: args.clear = True args.unmount = True args.reserve = args.setup args.mount = True - if not (args.show or args.mount or args.unmount or args.clear or args.reserve): - parser.error("no action specified") + if not (args.show or args.mount or args.unmount or + args.clear or args.reserve): + parser.error('no action specified') + # read huge page data from sysfs + hp_res = scan_huge_dirs(args.node) + + # read huge page mountpoint data + hp_mountpoint = args.directory + hp_mounted = hp_mountpoint in get_hugetlbfs_mountpoints() + hp_mount = HugepageMount(hp_mountpoint, hp_mounted) + + # get requested page size we will be working with if args.pagesize: pagesize_kb = get_memsize(args.pagesize) else: pagesize_kb = default_pagesize() - if not pagesize_kb: - sys.exit("Invalid page size: {}kB".format(pagesize_kb)) + # were we asked to clear? if args.clear: - clear_pages() + for hp in hp_res: + for sz in hp.valid_page_sizes: + hp[sz] = 0 + + # were we asked to unmount? if args.unmount: - umount_huge(args.directory) + hp_mount.unmount() + # were we asked to reserve pages? if args.reserve: - reserve_kb = get_memsize(args.reserve) - if reserve_kb % pagesize_kb != 0: - sys.exit( - 'Huge reservation {}kB is not a multiple of page size {}kB'. - format(reserve_kb, pagesize_kb)) - reserve_pages( - int(reserve_kb / pagesize_kb), pagesize_kb, node=args.node) + try_reserve_huge_pages(hp_res, args.reserve, pagesize_kb) + + # were we asked to mount? if args.mount: - mount_huge(pagesize_kb, args.directory, args.user, args.group) + hp_mount.mount(pagesize_kb, args.user, args.group) + + # were we asked to display status? if args.show: - show_pages() + print_hp_status(hp_res) print() - show_mount() + print_mount_status() -if __name__ == "__main__": - main() +if __name__ == '__main__': + try: + main() + except PermissionError: + sys.exit('Permission denied: need to be root!') + except subprocess.CalledProcessError as e: + sys.exit(f'Command failed: {e}') + except (KeyError, ValueError, OSError) as e: + sys.exit(f'Error: {e}') -- 2.43.5