From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id D399BA04C8; Fri, 18 Sep 2020 14:12:07 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id E4C311DA1C; Fri, 18 Sep 2020 14:11:56 +0200 (CEST) Received: from mga09.intel.com (mga09.intel.com [134.134.136.24]) by dpdk.org (Postfix) with ESMTP id 9E7CA1DA0E for ; Fri, 18 Sep 2020 14:11:53 +0200 (CEST) IronPort-SDR: GLMs37r0raqMpmh63YBqt+UIbsVwOBkdVYUhTdAuOXb3nzXOdnH03+V3AqBhruoB9vWIZ0Kpgj FZ3hm2GyvjFw== X-IronPort-AV: E=McAfee;i="6000,8403,9747"; a="160842566" X-IronPort-AV: E=Sophos;i="5.77,274,1596524400"; d="scan'208";a="160842566" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by orsmga102.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 18 Sep 2020 05:11:53 -0700 IronPort-SDR: 6tkZIPA5/RqiUIJRIvoZIad/lvTWpUva0kVQHASdPKpFKXLck7jIL4x80Dav+R855KByk91c0W fdNNMamFOSeQ== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.77,274,1596524400"; d="scan'208";a="410271212" Received: from silpixa00400466.ir.intel.com ([10.237.213.195]) by fmsmga001.fm.intel.com with ESMTP; 18 Sep 2020 05:11:50 -0700 From: Conor Walsh To: dev@dpdk.org Cc: david.marchand@redhat.com, ray.kinsella@intel.com, nhorman@tuxdriver.com, aconole@redhat.com, maicolgabriel@hotmail.com, thomas@monjalon.net, bruce.richardson@intel.com, anatoly.burakov@intel.com, Conor Walsh Date: Fri, 18 Sep 2020 12:11:35 +0000 Message-Id: <20200918121137.1370883-3-conor.walsh@intel.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20200918121137.1370883-1-conor.walsh@intel.com> References: <20200911160332.256343-1-conor.walsh@intel.com> <20200918121137.1370883-1-conor.walsh@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [dpdk-dev] [PATCH v4 2/4] devtools: add generation of compressed abi dump archives X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This patch adds a script that generates a compressed archive containing .dump files which can be used to perform ABI breakage checking for the build specified in the parameters. Invoke using "./gen-abi-tarball.py [-t ] [-a ] [-cf ]" - : dpdk tag e.g. "v20.11", default: latest - : required architecture e.g. "arm" or "x86_64", default: current system's architecture - : configuration file for cross compiling for another system, this flag is not required. e.g. "config/arm/arm64_armv8_linux_gcc", default: None E.g. "./gen-abi-tarball.py -t latest -a x86_64" If a compiler is not specified using the CC environmental variable then the script will default to using gcc. Using these parameters the script will produce a .tar.gz archive containing .dump files required to do ABI breakage checking Signed-off-by: Conor Walsh --- devtools/gen-abi-tarball.py | 142 ++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100755 devtools/gen-abi-tarball.py diff --git a/devtools/gen-abi-tarball.py b/devtools/gen-abi-tarball.py new file mode 100755 index 000000000..6c337104e --- /dev/null +++ b/devtools/gen-abi-tarball.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2020 Intel Corporation + +""" +This Python script generates a compressed archive containing .dump +files which can be used to perform ABI breakage checking for the +build specified in the parameters. +""" + +import os +from os.path import abspath, realpath, dirname, basename, join, getsize +import sys +import argparse +import platform +import tarfile +import subprocess +import shutil +import tempfile + +# Get command line options +def args_parse(): + parser = argparse.ArgumentParser( + description='This script is intended to generate ABI dump tarballs\n\n'+ + 'Supported environmental variables:\n'+ + '\t- CC: The required compiler will be determined using this environmental variable.\n', + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + '-t', '--tag', type=str, dest='tag', + help='DPDK tag e.g. latest or v20.11', default='latest') + parser.add_argument( + '-cf', '--cross-file', type=str, dest='crosscompile', + help='Set the location of a cross compile config') + parser.add_argument( + '-a', '--arch', type=str, dest='arch', + help='Architecture arm or x86_64', default=platform.machine()) + args = parser.parse_args() + return args + +# Function to execute git commands +def call_git(args): + args = list(filter(None, args)) + git_call = subprocess.run(['git'] + args, capture_output=True) + if git_call.returncode != 0: + print('ERROR Git returned an error', file=sys.stderr) + exit(1) + return git_call.stdout.decode('utf-8').strip() + +# Function to execute commands +def call_exec(args): + args = list(filter(None, args)) + exec_call = subprocess.run(args, stdout=subprocess.DEVNULL) + if exec_call.returncode != 0: + print('ERROR Script returned an error', file=sys.stderr) + exit(1) + +# Get the required git tag +def get_tag(tag): + tags = call_git(['ls-remote', '--tags', 'http://dpdk.org/git/dpdk']).split('\n') + tags = [t.split('/')[-1].strip() for t in tags if 'rc' not in t and not t.endswith('{}')] + if tag == 'latest': + tag = tags[-1] + if tag not in tags: + print('ERROR supplied tag does not exist in DPDK repo', file=sys.stderr) + exit(1) + return tag + +def main(): + args = args_parse() + + # Get the cross-compile option + cross_comp_meson = [None, None] + if args.crosscompile: + cross_comp_meson = ['--cross-file', abspath(args.crosscompile)] + + tag = get_tag(args.tag) + + # Get the specified compiler from system + if 'CC' in os.environ: + comp = os.environ['CC'] + else: + print('No compiler specified in environmental varibles, setting CC=gcc') + comp = 'gcc' + os.environ['CC'] = 'gcc' + + # Print the configuration to the user + print('\nSelected Build: {}, Compiler: {}, Architecture: {}, Cross Compile: {}'.format(tag,comp,args.arch,cross_comp_meson[1])) + + # Store the users working directory + baseDir = os.getcwd() + # Store devtools dir + devtoolsDir = abspath(dirname(realpath(sys.argv[0]))) + + # Create directory for DPDK git repo and build + tmpDir = tempfile.TemporaryDirectory(dir = "/tmp") + + os.chdir(tmpDir.name) + # Clone DPDK and switch to specified tag + print('Cloning {} from DPDK git'.format(tag)) + call_git(['clone', '--quiet', 'http://dpdk.org/git/dpdk']) + os.chdir('dpdk') + call_git(['checkout', '--quiet', tag]) + + # Create build folder with meson and set debug build and cross compile (if needed) + print('Configuring Meson') + call_exec(['meson', '-Dbuildtype=debug', 'dumpbuild'] + cross_comp_meson) + #os.system('meson -Dbuildtype=debug dumpbuild {} >/dev/null'.format(cross_comp_meson)) + print('Building DPDK . . .') + #Build DPDK with ninja + call_exec(['ninja', '-C', 'dumpbuild']) + + # Create dump files and output to dump directory + dumpDir = join(baseDir,'{}-{}-{}-abi_dump'.format(tag,comp,args.arch)) + print('Generating ABI dump files') + call_exec([join(devtoolsDir,'gen-abi.sh'), 'dumpbuild']) + try: + shutil.copytree('dumpbuild/dump', dumpDir) + except FileExistsError as error: + print('ERROR The {} directory already exists, ensure it is not present before running script'.format(dumpDir), file=sys.stderr) + tmpDir.cleanup() + exit(1) + + # Compress the dump directory + print('Creating Tarball of dump files') + os.chdir(baseDir) + origSize = 0 + for f in os.scandir(dumpDir): + origSize += getsize(f) + with tarfile.open('{}.tar.gz'.format(dumpDir), "w:gz") as tar: + tar.add(dumpDir, arcname=basename(dumpDir)) + newSize = getsize('{}.tar.gz'.format(dumpDir)) + + # Remove all temporary directories + print('Cleaning up temporary directories') + shutil.rmtree(dumpDir) + tmpDir.cleanup() + + #Print output of the script to the user + print('\nDump of DPDK ABI {} is available in {}.tar.gz (Original Size: {:.1f}MB, Compressed Size: {:.1f}MB)\n'.format(tag,dumpDir.split('/')[-1],float(origSize)*1e-6,float(newSize)*1e-6)) + +if __name__ == "__main__": + main() -- 2.25.1