Re: [dpdk-dev] [PATCH v3 2/4] devtools: add generation of compressed abi dump archives

["Burakov, Anatoly" <anatoly.burakov@intel.com>]

On 11-Sep-20 5:03 PM, Conor Walsh wrote:
> This patch adds a script that generates a compressed archive
> containing .dump files which can be used to perform ABI
> breakage checking for the build specified in the parameters.
> Invoke using "./gen-abi-tarball.py -t <tag> -a <arch> [-cf <cross-file>]"
>   - <tag>: dpdk tag e.g. "v20.11"
>   - <arch>: required architecture e.g. "arm" or "x86_64"
>   - <cross-file>: configuration file for cross compiling for another
>                   system, this flag is not required.
>                   e.g. "config/arm/arm64_armv8_linux_gcc"
> E.g. "./gen-abi-tarball.py -t latest -a x86_64"
> If a compiler is not specified using the CC environmental variable then
> the script will default to using gcc.
> Using these parameters the script will produce a .tar.gz archive
> containing .dump files required to do ABI breakage checking
> 
> Signed-off-by: Conor Walsh <conor.walsh@intel.com>
> ---

Just a general comment: this script looks like it's bash translated to 
python. If you're going to do that, you might as well just write it in bash?

>   devtools/gen-abi-tarball.py | 125 ++++++++++++++++++++++++++++++++++++
>   1 file changed, 125 insertions(+)
>   create mode 100755 devtools/gen-abi-tarball.py
> 
> diff --git a/devtools/gen-abi-tarball.py b/devtools/gen-abi-tarball.py
> new file mode 100755
> index 000000000..06761fca6
> --- /dev/null
> +++ b/devtools/gen-abi-tarball.py
> @@ -0,0 +1,125 @@
> +#!/usr/bin/env python3
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2020 Intel Corporation
> +
> +import sys
> +import os
> +import argparse
> +

It is preferred to wrap executable code in "if __name__ == 
"__main__"..., helps with importing code while avoiding executing it on 
import.

> +# Get command line arguments
> +parser = argparse.ArgumentParser(usage='\rThis script is intended to generate ABI dump tarballs\n'+
> +                                       'Supported environmental variables\n'+
> +                                       '\t- CC: The required compiler will be determined using this environmental variable.\n')
> +parser.add_argument('-t', '--tag', type=str, dest='tag', help='DPDK tag e.g. latest or v20.11')
> +parser.add_argument('-cf', '--cross-file', type=str, dest='crosscompile', help='Set the location of a cross compile config')
> +parser.add_argument('-a', '--arch', type=str, dest='arch', help='Arch arm or x86_64')
> +args = parser.parse_args()
> +
> +# Get the DPDK tag if not supplied set as latest
> +if args.tag:
> +    user_tag = args.tag
> +else:
> +    user_tag = 'latest'
> +    print('No tag supplied defaulting to latest')

There's a "default" option for arguments.

> +
> +# Get the cross-compile option
> +if args.crosscompile:
> +    cross_comp = args.crosscompile
> +    if not args.arch:
> +        sys.stderr.write('ERROR Arch must be set using -a when using cross compile\n')
> +        exit(1)
> +    cross_comp = os.path.abspath(cross_comp)
> +    cross_comp_meson = '--cross-file '+cross_comp
> +else:
> +    cross_comp = ''
> +    cross_comp_meson = ''
> +
> +# Get the required system architecture if not supplied set as x86_64
> +if args.arch:
> +    arch = args.arch
> +else:
> +    arch = os.popen('uname -m').read().strip()

There's a built-in python library for this, i think it's called "platform".

> +    print('No system architecture supplied defaulting to '+arch)
> +
> +tag = ''
> +# If the user did not supply tag or wants latest then get latest tag
> +if user_tag == 'latest':
> +    # Get latest quarterly build tag from git repo
> +    tag = os.popen('git ls-remote --tags http://dpdk.org/git/dpdk | grep -v "rc\|{}" | tail -n 1 | sed "s/.*\///"').read().strip()
> +else:
> +    tag = user_tag
> +    # If the user supplied tag is not in the DPDK repo then fail
> +    tag_check = int(os.popen('git ls-remote http://dpdk.org/git/dpdk refs/tags/'+tag+' | wc -l').read().strip())
> +    if tag_check != 1:
> +        sys.stderr.write('ERROR supplied tag does not exist in DPDK repo\n')
> +        exit(1)
> +
> +# Get the specified compiler from system
> +comp_env = 'CC'
> +if comp_env in os.environ:
> +    comp = os.environ[comp_env]
> +    comp_default = ''
> +else:
> +    print('No compiler specified, defaulting to gcc')
> +    comp = 'gcc'
> +    comp_default = 'CC=gcc'
> +
> +# Print the configuration to the user
> +print('\nSelected Build: '+tag+', Compiler: '+comp+', Architecture: '+arch+', Cross Compile: '+cross_comp)

Here and in similar places: please don't use string concatenation, use 
string formatting instead.

> +
> +# Store the base directory script is working from
> +baseDir = os.getcwd()
> +# Store devtools dir
> +devtoolsDir = os.path.abspath(os.path.dirname(os.path.realpath(sys.argv[0])))
> +
> +# Create directory for DPDK git repo and build
> +try:
> +    os.mkdir('dump_dpdk')
> +except OSError as error:
> +    sys.stderr.write('ERROR The dump_dpdk directory could not be created, ensure it does not exist before start\n')
> +    exit(1)
> +os.chdir('dump_dpdk')
> +# Clone DPDK and switch to specified tag
> +print('Cloning '+tag+' from DPDK git')
> +os.popen('git clone --quiet http://dpdk.org/git/dpdk >/dev/null').read()
> +os.chdir('dpdk')
> +os.popen('git checkout --quiet '+tag+' >/dev/null').read()
> +
> +# Create build folder with meson and set debug build and cross compile (if needed)
> +print('Configuring Meson')
> +os.popen(comp_default+' meson dumpbuild '+cross_comp_meson+' >/dev/null').read()

You do this os.popen(something > /dev/bull).read() quite a lot, maybe 
put it into a function? Also, since you're discarding the data anyway - 
why os.popen().read() instead of os.system()?

> +os.chdir('dumpbuild')
> +os.popen('meson configure -Dbuildtype=debug >/dev/null').read()
> +print('Building DPDK . . .')
> +#Build DPDK with ninja
> +os.popen('ninja >/dev/null').read()
> +gccDir = os.getcwd()
> +
> +# Create directory for abi dump files
> +dumpDir = os.path.join(baseDir,tag+'-'+comp+'-'+arch+'-abi_dump')
> +try:
> +    os.mkdir(dumpDir)
> +except OSError as error:
> +    sys.stderr.write('ERROR The '+dumpDir+' directory could not be created\n')
> +    os.popen('rm -rf '+os.path.join(baseDir,'dump_dpdk')).read()
> +    exit(1)
> +
> +# Create dump files and output to dump directory
> +print('Generating ABI dump files')
> +genabiout = os.popen(os.path.join(devtoolsDir,'gen-abi.sh')+' '+gccDir).read()
> +os.popen('cp dump/* '+dumpDir).read()
> +
> +# Compress the dump directory
> +print('Creating Tarball of dump files')
> +os.chdir(baseDir)
> +origSize = os.popen('du -sh '+dumpDir+' | sed "s/\s.*$//"').read()
> +os.popen('tar -czf '+dumpDir.split('/')[-1]+'.tar.gz '+dumpDir.split('/')[-1]+' >/dev/null').read()
> +newSize = os.popen('du -sh '+dumpDir+'.tar.gz | sed "s/\s.*$//"').read()
> +
> +# Remove all temporary directories
> +print('Cleaning up temporary directories')
> +os.popen('rm -rf '+dumpDir).read()
> +os.popen('rm -rf '+os.path.join(baseDir,'dump_dpdk')).read()
> +
> +#Print output of the script to the user
> +print('\nDump of DPDK ABI '+tag+' is available in '+dumpDir.split('/')[-1]+'.tar.gz (Original Size: '+origSize.strip()+', Compressed Size:'+newSize.strip()+')\n')
> 


-- 
Thanks,
Anatoly