From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 5BFC8A04C7; Mon, 14 Sep 2020 14:50:20 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 986B5FFA; Mon, 14 Sep 2020 14:50:19 +0200 (CEST) Received: from mga06.intel.com (mga06.intel.com [134.134.136.31]) by dpdk.org (Postfix) with ESMTP id DD015E07 for ; Mon, 14 Sep 2020 14:50:17 +0200 (CEST) IronPort-SDR: s1y0VC7+aE8qA1kXDOVMPKjdpdNprZCSCqFODb7ZG9OlWroAUuyA5XchavrfV9I57hfwjomuUT B5Mx9iiPjMeg== X-IronPort-AV: E=McAfee;i="6000,8403,9743"; a="220622172" X-IronPort-AV: E=Sophos;i="5.76,426,1592895600"; d="scan'208";a="220622172" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga005.jf.intel.com ([10.7.209.41]) by orsmga104.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 14 Sep 2020 05:50:16 -0700 IronPort-SDR: gv+aB6QA8dti8anbK107whJt/SLgxXQRSmjJUQKjBJX8a+BeyGyvPu2f7Zb7fyY5JZE4j4VZqW UBmZD4Snf2PQ== X-IronPort-AV: E=Sophos;i="5.76,426,1592895600"; d="scan'208";a="482329084" Received: from aburakov-mobl.ger.corp.intel.com (HELO [10.251.82.108]) ([10.251.82.108]) by orsmga005-auth.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 14 Sep 2020 05:50:14 -0700 To: Conor Walsh , dev@dpdk.org Cc: david.marchand@redhat.com, ray.kinsella@intel.com, nhorman@tuxdriver.com, aconole@redhat.com, maicolgabriel@hotmail.com, thomas@monjalon.net, bruce.richardson@intel.com References: <20200910142121.3995680-1-conor.walsh@intel.com> <20200911160332.256343-1-conor.walsh@intel.com> <20200911160332.256343-3-conor.walsh@intel.com> From: "Burakov, Anatoly" Message-ID: Date: Mon, 14 Sep 2020 13:50:12 +0100 User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:68.0) Gecko/20100101 Thunderbird/68.12.0 MIME-Version: 1.0 In-Reply-To: <20200911160332.256343-3-conor.walsh@intel.com> Content-Type: text/plain; charset=utf-8; format=flowed Content-Language: en-US Content-Transfer-Encoding: 7bit Subject: Re: [dpdk-dev] [PATCH v3 2/4] devtools: add generation of compressed abi dump archives X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" On 11-Sep-20 5:03 PM, Conor Walsh wrote: > This patch adds a script that generates a compressed archive > containing .dump files which can be used to perform ABI > breakage checking for the build specified in the parameters. > Invoke using "./gen-abi-tarball.py -t -a [-cf ]" > - : dpdk tag e.g. "v20.11" > - : required architecture e.g. "arm" or "x86_64" > - : configuration file for cross compiling for another > system, this flag is not required. > e.g. "config/arm/arm64_armv8_linux_gcc" > E.g. "./gen-abi-tarball.py -t latest -a x86_64" > If a compiler is not specified using the CC environmental variable then > the script will default to using gcc. > Using these parameters the script will produce a .tar.gz archive > containing .dump files required to do ABI breakage checking > > Signed-off-by: Conor Walsh > --- Just a general comment: this script looks like it's bash translated to python. If you're going to do that, you might as well just write it in bash? > devtools/gen-abi-tarball.py | 125 ++++++++++++++++++++++++++++++++++++ > 1 file changed, 125 insertions(+) > create mode 100755 devtools/gen-abi-tarball.py > > diff --git a/devtools/gen-abi-tarball.py b/devtools/gen-abi-tarball.py > new file mode 100755 > index 000000000..06761fca6 > --- /dev/null > +++ b/devtools/gen-abi-tarball.py > @@ -0,0 +1,125 @@ > +#!/usr/bin/env python3 > +# SPDX-License-Identifier: BSD-3-Clause > +# Copyright(c) 2020 Intel Corporation > + > +import sys > +import os > +import argparse > + It is preferred to wrap executable code in "if __name__ == "__main__"..., helps with importing code while avoiding executing it on import. > +# Get command line arguments > +parser = argparse.ArgumentParser(usage='\rThis script is intended to generate ABI dump tarballs\n'+ > + 'Supported environmental variables\n'+ > + '\t- CC: The required compiler will be determined using this environmental variable.\n') > +parser.add_argument('-t', '--tag', type=str, dest='tag', help='DPDK tag e.g. latest or v20.11') > +parser.add_argument('-cf', '--cross-file', type=str, dest='crosscompile', help='Set the location of a cross compile config') > +parser.add_argument('-a', '--arch', type=str, dest='arch', help='Arch arm or x86_64') > +args = parser.parse_args() > + > +# Get the DPDK tag if not supplied set as latest > +if args.tag: > + user_tag = args.tag > +else: > + user_tag = 'latest' > + print('No tag supplied defaulting to latest') There's a "default" option for arguments. > + > +# Get the cross-compile option > +if args.crosscompile: > + cross_comp = args.crosscompile > + if not args.arch: > + sys.stderr.write('ERROR Arch must be set using -a when using cross compile\n') > + exit(1) > + cross_comp = os.path.abspath(cross_comp) > + cross_comp_meson = '--cross-file '+cross_comp > +else: > + cross_comp = '' > + cross_comp_meson = '' > + > +# Get the required system architecture if not supplied set as x86_64 > +if args.arch: > + arch = args.arch > +else: > + arch = os.popen('uname -m').read().strip() There's a built-in python library for this, i think it's called "platform". > + print('No system architecture supplied defaulting to '+arch) > + > +tag = '' > +# If the user did not supply tag or wants latest then get latest tag > +if user_tag == 'latest': > + # Get latest quarterly build tag from git repo > + tag = os.popen('git ls-remote --tags http://dpdk.org/git/dpdk | grep -v "rc\|{}" | tail -n 1 | sed "s/.*\///"').read().strip() > +else: > + tag = user_tag > + # If the user supplied tag is not in the DPDK repo then fail > + tag_check = int(os.popen('git ls-remote http://dpdk.org/git/dpdk refs/tags/'+tag+' | wc -l').read().strip()) > + if tag_check != 1: > + sys.stderr.write('ERROR supplied tag does not exist in DPDK repo\n') > + exit(1) > + > +# Get the specified compiler from system > +comp_env = 'CC' > +if comp_env in os.environ: > + comp = os.environ[comp_env] > + comp_default = '' > +else: > + print('No compiler specified, defaulting to gcc') > + comp = 'gcc' > + comp_default = 'CC=gcc' > + > +# Print the configuration to the user > +print('\nSelected Build: '+tag+', Compiler: '+comp+', Architecture: '+arch+', Cross Compile: '+cross_comp) Here and in similar places: please don't use string concatenation, use string formatting instead. > + > +# Store the base directory script is working from > +baseDir = os.getcwd() > +# Store devtools dir > +devtoolsDir = os.path.abspath(os.path.dirname(os.path.realpath(sys.argv[0]))) > + > +# Create directory for DPDK git repo and build > +try: > + os.mkdir('dump_dpdk') > +except OSError as error: > + sys.stderr.write('ERROR The dump_dpdk directory could not be created, ensure it does not exist before start\n') > + exit(1) > +os.chdir('dump_dpdk') > +# Clone DPDK and switch to specified tag > +print('Cloning '+tag+' from DPDK git') > +os.popen('git clone --quiet http://dpdk.org/git/dpdk >/dev/null').read() > +os.chdir('dpdk') > +os.popen('git checkout --quiet '+tag+' >/dev/null').read() > + > +# Create build folder with meson and set debug build and cross compile (if needed) > +print('Configuring Meson') > +os.popen(comp_default+' meson dumpbuild '+cross_comp_meson+' >/dev/null').read() You do this os.popen(something > /dev/bull).read() quite a lot, maybe put it into a function? Also, since you're discarding the data anyway - why os.popen().read() instead of os.system()? > +os.chdir('dumpbuild') > +os.popen('meson configure -Dbuildtype=debug >/dev/null').read() > +print('Building DPDK . . .') > +#Build DPDK with ninja > +os.popen('ninja >/dev/null').read() > +gccDir = os.getcwd() > + > +# Create directory for abi dump files > +dumpDir = os.path.join(baseDir,tag+'-'+comp+'-'+arch+'-abi_dump') > +try: > + os.mkdir(dumpDir) > +except OSError as error: > + sys.stderr.write('ERROR The '+dumpDir+' directory could not be created\n') > + os.popen('rm -rf '+os.path.join(baseDir,'dump_dpdk')).read() > + exit(1) > + > +# Create dump files and output to dump directory > +print('Generating ABI dump files') > +genabiout = os.popen(os.path.join(devtoolsDir,'gen-abi.sh')+' '+gccDir).read() > +os.popen('cp dump/* '+dumpDir).read() > + > +# Compress the dump directory > +print('Creating Tarball of dump files') > +os.chdir(baseDir) > +origSize = os.popen('du -sh '+dumpDir+' | sed "s/\s.*$//"').read() > +os.popen('tar -czf '+dumpDir.split('/')[-1]+'.tar.gz '+dumpDir.split('/')[-1]+' >/dev/null').read() > +newSize = os.popen('du -sh '+dumpDir+'.tar.gz | sed "s/\s.*$//"').read() > + > +# Remove all temporary directories > +print('Cleaning up temporary directories') > +os.popen('rm -rf '+dumpDir).read() > +os.popen('rm -rf '+os.path.join(baseDir,'dump_dpdk')).read() > + > +#Print output of the script to the user > +print('\nDump of DPDK ABI '+tag+' is available in '+dumpDir.split('/')[-1]+'.tar.gz (Original Size: '+origSize.strip()+', Compressed Size:'+newSize.strip()+')\n') > -- Thanks, Anatoly