Script to track growth of stable and experimental symbols over releases since v19.11. Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/count_symbols.py | 230 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100755 devtools/count_symbols.py diff --git a/devtools/count_symbols.py b/devtools/count_symbols.py new file mode 100755 index 0000000000..7b29651044 --- /dev/null +++ b/devtools/count_symbols.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +from pathlib import Path +import sys, os +import subprocess +import argparse +import re +import datetime + +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + exit() + +symbolMapGrammar = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" + +#abi_ver = ['21', '20.0.1', '20.0', '20'] + +def get_abi_versions(): + year = datetime.date.today().year - 2000 + s=" |".join(['\'{}\''.format(i) for i in reversed(range(21, year + 1)) ]) + s = s + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return s + +def get_dpdk_releases(): + year = datetime.date.today().year - 2000 + s="|".join("{}".format(i) for i in range(19,year + 1)) + pattern = re.compile('^\"v(' + s + ')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + if result.stderr.startswith(b'fatal'): + result = None + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [ tag.replace('\"','') \ + for tag in reversed(tags) \ + if pattern.match(tag) ][:-3] + + return tags + + +def get_terminal_rows(): + rows, _ = os.popen('stty size', 'r').read().split() + return int(rows) + +def fix_directory_name(path): + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + +# fix removal of the librte_ from the directory names +def directory_renamed(path, rel): + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + if result.stderr.startswith(b'fatal'): + result = None + + return result + +# fix renaming of map files +def mapfile_renamed(path, rel): + newfile = None + + result = subprocess.run(['git', 'ls-tree', \ + rel, str(path.parent) + '/'], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if(newfile is not None): + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + if result.stderr.startswith(b'fatal'): + result = None + + else: + result = None + + return result + +# renaming of the map file & renaming of directory +def mapfile_and_directory_renamed(path, rel): + mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name)) + + return mapfile_renamed(mapfilepath, rel) + +fix_strategies = [directory_renamed, \ + mapfile_renamed, \ + mapfile_and_directory_renamed] + +fmt = col_fmt = "" + +def set_terminal_output(dpdk_rel): + global fmt, col_fmt + + fmt = '{:<50}' + col_fmt = fmt + for rel in dpdk_rel: + fmt += '{:<6}{:<6}' + col_fmt += '{:<12}' + +def set_csv_output(dpdk_rel): + global fmt, col_fmt + + fmt = '{},' + col_fmt = fmt + for rel in dpdk_rel: + fmt += '{},{},' + col_fmt += '{},,' + +output_formats = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } +directories = 'drivers, lib' + +def main(): + global fmt, col_fmt, symbolMapGrammar + + parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs') + parser.add_argument('--format-output', choices=['terminal','csv'], \ + default='terminal') + parser.add_argument('--directory', choices=directories, + default=directories) + args = parser.parse_args() + + dpdk_rel = get_dpdk_releases() + + # set the output format + output_formats[args.format_output](dpdk_rel) + + column_titles = ['mapfile'] + dpdk_rel + print(col_fmt.format(*column_titles)) + + symbolMapGrammar = symbolMapGrammar.format(get_abi_versions()) + MAPParser = makeGrammar(symbolMapGrammar, {}) + + terminal_rows = get_terminal_rows() + row = 0 + + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + csym = [0] * 2 + relsym = [str(path)] + + for rel in dpdk_rel: + i = csym[0] = csym[1] = 0 + abi_sections = None + + tagfile = '{}:{}'.format(rel,path) + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + + if result.stderr.startswith(b'fatal'): + result = None + + while(result is None and i < len(fix_strategies)): + result = fix_strategies[i](path, rel) + i += 1 + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = MAPParser(mapfile).abi() + + if abi_sections is not None: + # which versions are present, and we care about + ignore = ['EXPERIMENTAL','INTERNAL'] + found_ver = [ver \ + for ver in abi_sections \ + if ver not in ignore] + + for ver in found_ver: + csym[0] += len(abi_sections[ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in abi_sections: + csym[1] = len(abi_sections['EXPERIMENTAL']) + + relsym += csym + + print(fmt.format(*relsym)) + row += 1 + + if((terminal_rows>0) and ((row % terminal_rows) == 0)): + print(col_fmt.format(*column_titles)) + +if __name__ == '__main__': + main() -- 2.26.2
On Fri, 18 Jun 2021 17:36:59 +0100
Ray Kinsella <mdr@ashroe.eu> wrote:
> Script to track growth of stable and experimental symbols
> over releases since v19.11.
>
> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
pylint reports some things that should be fixed. Don't worry about the naming style
and docstring but others should be addressed.
************* Module count_symbols
devtools/count_symbols.py:12:0: W0311: Bad indentation. Found 8 spaces, expected 4 (bad-indentation)
devtools/count_symbols.py:14:0: W0311: Bad indentation. Found 8 spaces, expected 4 (bad-indentation)
devtools/count_symbols.py:16:0: W0311: Bad indentation. Found 8 spaces, expected 4 (bad-indentation)
devtools/count_symbols.py:109:0: C0325: Unnecessary parens after 'if' keyword (superfluous-parens)
devtools/count_symbols.py:230:0: W0311: Bad indentation. Found 8 spaces, expected 4 (bad-indentation)
devtools/count_symbols.py:47:41: W1401: Anomalous backslash in string: '\.'. String constant might be missing an r prefix. (anomalous-backslash-in-string)
devtools/count_symbols.py:47:43: W1401: Anomalous backslash in string: '\d'. String constant might be missing an r prefix. (anomalous-backslash-in-string)
devtools/count_symbols.py:1:0: C0114: Missing module docstring (missing-module-docstring)
devtools/count_symbols.py:5:0: C0410: Multiple imports on one line (sys, os) (multiple-imports)
devtools/count_symbols.py:16:8: R1722: Consider using sys.exit() (consider-using-sys-exit)
devtools/count_symbols.py:18:0: C0103: Constant name "symbolMapGrammar" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:37:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:39:4: C0103: Variable name "s" doesn't conform to snake_case naming style (invalid-name)
devtools/count_symbols.py:40:4: C0103: Variable name "s" doesn't conform to snake_case naming style (invalid-name)
devtools/count_symbols.py:44:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:46:4: C0103: Variable name "s" doesn't conform to snake_case naming style (invalid-name)
devtools/count_symbols.py:50:13: W1510: Using subprocess.run without explicitly set `check` is not recommended. (subprocess-run-check)
devtools/count_symbols.py:66:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:70:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:78:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:82:13: W1510: Using subprocess.run without explicitly set `check` is not recommended. (subprocess-run-check)
devtools/count_symbols.py:91:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:94:13: W1510: Using subprocess.run without explicitly set `check` is not recommended. (subprocess-run-check)
devtools/count_symbols.py:112:17: W1510: Using subprocess.run without explicitly set `check` is not recommended. (subprocess-run-check)
devtools/count_symbols.py:124:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:133:0: C0103: Constant name "fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:133:6: C0103: Constant name "col_fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:135:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:136:4: W0603: Using the global statement (global-statement)
devtools/count_symbols.py:136:4: C0103: Constant name "fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:136:4: C0103: Constant name "col_fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:140:8: W0612: Unused variable 'rel' (unused-variable)
devtools/count_symbols.py:144:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:145:4: W0603: Using the global statement (global-statement)
devtools/count_symbols.py:145:4: C0103: Constant name "fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:145:4: C0103: Constant name "col_fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:149:8: W0612: Unused variable 'rel' (unused-variable)
devtools/count_symbols.py:156:0: C0103: Constant name "directories" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:158:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:158:0: R0914: Too many local variables (20/15) (too-many-locals)
devtools/count_symbols.py:159:4: W0603: Using the global statement (global-statement)
devtools/count_symbols.py:159:4: C0103: Constant name "fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:159:4: C0103: Constant name "col_fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:159:4: C0103: Constant name "symbolMapGrammar" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:177:4: C0103: Variable name "MAPParser" doesn't conform to snake_case naming style (invalid-name)
devtools/count_symbols.py:192:25: W1510: Using subprocess.run without explicitly set `check` is not recommended. (subprocess-run-check)
devtools/count_symbols.py:5:0: W0611: Unused import sys (unused-import)
-----------------------------------
Your code has been rated at 6.27/10
>
> pylint reports some things that should be fixed. Don't worry about the naming style
> and docstring but others should be addressed.
[SNIP]
Ah, rookie mistake,
I ran checkpatch and thought that I was all good.
I will sort it out thanks.
Ray K
Script to track growth of stable and experimental symbols over releases since v19.11. Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/count_symbols.py | 230 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100755 devtools/count_symbols.py diff --git a/devtools/count_symbols.py b/devtools/count_symbols.py new file mode 100755 index 0000000000..7b29651044 --- /dev/null +++ b/devtools/count_symbols.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +from pathlib import Path +import sys, os +import subprocess +import argparse +import re +import datetime + +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + exit() + +symbolMapGrammar = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" + +#abi_ver = ['21', '20.0.1', '20.0', '20'] + +def get_abi_versions(): + year = datetime.date.today().year - 2000 + s=" |".join(['\'{}\''.format(i) for i in reversed(range(21, year + 1)) ]) + s = s + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return s + +def get_dpdk_releases(): + year = datetime.date.today().year - 2000 + s="|".join("{}".format(i) for i in range(19,year + 1)) + pattern = re.compile('^\"v(' + s + ')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + if result.stderr.startswith(b'fatal'): + result = None + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [ tag.replace('\"','') \ + for tag in reversed(tags) \ + if pattern.match(tag) ][:-3] + + return tags + + +def get_terminal_rows(): + rows, _ = os.popen('stty size', 'r').read().split() + return int(rows) + +def fix_directory_name(path): + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + +# fix removal of the librte_ from the directory names +def directory_renamed(path, rel): + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + if result.stderr.startswith(b'fatal'): + result = None + + return result + +# fix renaming of map files +def mapfile_renamed(path, rel): + newfile = None + + result = subprocess.run(['git', 'ls-tree', \ + rel, str(path.parent) + '/'], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if(newfile is not None): + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + if result.stderr.startswith(b'fatal'): + result = None + + else: + result = None + + return result + +# renaming of the map file & renaming of directory +def mapfile_and_directory_renamed(path, rel): + mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name)) + + return mapfile_renamed(mapfilepath, rel) + +fix_strategies = [directory_renamed, \ + mapfile_renamed, \ + mapfile_and_directory_renamed] + +fmt = col_fmt = "" + +def set_terminal_output(dpdk_rel): + global fmt, col_fmt + + fmt = '{:<50}' + col_fmt = fmt + for rel in dpdk_rel: + fmt += '{:<6}{:<6}' + col_fmt += '{:<12}' + +def set_csv_output(dpdk_rel): + global fmt, col_fmt + + fmt = '{},' + col_fmt = fmt + for rel in dpdk_rel: + fmt += '{},{},' + col_fmt += '{},,' + +output_formats = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } +directories = 'drivers, lib' + +def main(): + global fmt, col_fmt, symbolMapGrammar + + parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs') + parser.add_argument('--format-output', choices=['terminal','csv'], \ + default='terminal') + parser.add_argument('--directory', choices=directories, + default=directories) + args = parser.parse_args() + + dpdk_rel = get_dpdk_releases() + + # set the output format + output_formats[args.format_output](dpdk_rel) + + column_titles = ['mapfile'] + dpdk_rel + print(col_fmt.format(*column_titles)) + + symbolMapGrammar = symbolMapGrammar.format(get_abi_versions()) + MAPParser = makeGrammar(symbolMapGrammar, {}) + + terminal_rows = get_terminal_rows() + row = 0 + + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + csym = [0] * 2 + relsym = [str(path)] + + for rel in dpdk_rel: + i = csym[0] = csym[1] = 0 + abi_sections = None + + tagfile = '{}:{}'.format(rel,path) + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + + if result.stderr.startswith(b'fatal'): + result = None + + while(result is None and i < len(fix_strategies)): + result = fix_strategies[i](path, rel) + i += 1 + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = MAPParser(mapfile).abi() + + if abi_sections is not None: + # which versions are present, and we care about + ignore = ['EXPERIMENTAL','INTERNAL'] + found_ver = [ver \ + for ver in abi_sections \ + if ver not in ignore] + + for ver in found_ver: + csym[0] += len(abi_sections[ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in abi_sections: + csym[1] = len(abi_sections['EXPERIMENTAL']) + + relsym += csym + + print(fmt.format(*relsym)) + row += 1 + + if((terminal_rows>0) and ((row % terminal_rows) == 0)): + print(col_fmt.format(*column_titles)) + +if __name__ == '__main__': + main() -- 2.26.2
Script to track growth of stable and experimental symbols over releases since v19.11. Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- v2: reworked to fix pylint errors v3: sent with the current in-reply-to devtools/count_symbols.py | 262 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100755 devtools/count_symbols.py diff --git a/devtools/count_symbols.py b/devtools/count_symbols.py new file mode 100755 index 0000000000..30be09754f --- /dev/null +++ b/devtools/count_symbols.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +'''Tool to count the number of symbols in each DPDK release''' +from pathlib import Path +import sys +import os +import subprocess +import argparse +import re +import datetime + +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + sys.exit() + +MAP_GRAMMAR = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" + +def get_abi_versions(): + '''Returns a string of possible dpdk abi versions''' + + year = datetime.date.today().year - 2000 + tags = " |".join(['\'{}\''.format(i) \ + for i in reversed(range(21, year + 1)) ]) + tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return tags + +def get_dpdk_releases(): + '''Returns a list of dpdk release tags names since v19.11''' + + year = datetime.date.today().year - 2000 + year_range = "|".join("{}".format(i) for i in range(19,year + 1)) + pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + try: + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + print("Failed to interogate git for release tags") + sys.exit() + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [ tag.replace('\"','') \ + for tag in reversed(tags) \ + if pattern.match(tag) ][:-3] + + return tags + +def fix_directory_name(path): + '''Prepend librte to the source directory name''' + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + +def directory_renamed(path, rel): + '''Fix removal of the librte_ from the directory names''' + + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + return result + +def mapfile_renamed(path, rel): + '''Fix renaming of map files''' + newfile = None + + result = subprocess.run(['git', 'ls-tree', \ + rel, str(path.parent) + '/'], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if newfile is not None: + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + else: + result = None + + return result + +def mapfile_and_directory_renamed(path, rel): + '''Fix renaming of the map file & the source directory''' + mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name)) + + return mapfile_renamed(mapfilepath, rel) + +def get_terminal_rows(): + '''Find the number of rows in the terminal''' + + rows, _ = os.popen('stty size', 'r').read().split() + return int(rows) + +class FormatOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.OUTPUT_FORMATS[format_output](self,dpdk_releases) + self.column_titles = ['mapfile'] + dpdk_releases + + self.terminal_rows = get_terminal_rows() + self.row = 0 + + def set_terminal_output(self,dpdk_rel): + '''Set the output format to Tabbed Seperated Values''' + + self.output_fmt = '{:<50}' + \ + ''.join(['{:<6}{:<6}'] * (len(dpdk_rel))) + self.column_fmt = '{:50}' + \ + ''.join(['{:<12}'] * (len(dpdk_rel))) + + def set_csv_output(self,dpdk_rel): + '''Set the output format to Comma Seperated Values''' + + self.output_fmt = '{},' + \ + ','.join(['{},{}'] * (len(dpdk_rel))) + self.column_fmt = '{},' + \ + ','.join(['{},'] * (len(dpdk_rel))) + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + self.row += 1 + + def print_row(self,symbols): + '''Print row of symbol values''' + print(self.output_fmt.format(*symbols)) + self.row += 1 + + if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)): + self.print_columns() + + OUTPUT_FORMATS = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } + +SRC_DIRECTORIES = 'drivers, lib' +IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL'] +FIX_STRATEGIES = [directory_renamed, \ + mapfile_renamed, \ + mapfile_and_directory_renamed] + +def count_release_symbols(map_parser, release, mapfile_path): + '''Count the symbols for a given release and mapfile''' + csym = [0] * 2 + abi_sections = None + + tagfile = '{}:{}'.format(release,mapfile_path) + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + for fix_strategy in FIX_STRATEGIES: + if result is not None: + break + result = fix_strategy(mapfile_path, release) + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = map_parser(mapfile).abi() + + if abi_sections is not None: + # which versions are present, and we care about + found_ver = [ver \ + for ver in abi_sections \ + if ver not in IGNORE_SECTIONS] + + for ver in found_ver: + csym[0] += len(abi_sections[ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in abi_sections: + csym[1] = len(abi_sections['EXPERIMENTAL']) + + return csym + +def main(): + '''Main entry point''' + + parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs') + parser.add_argument('--format-output', choices=['terminal','csv'], \ + default='terminal') + parser.add_argument('--directory', choices=SRC_DIRECTORIES, + default=SRC_DIRECTORIES) + args = parser.parse_args() + + dpdk_releases = get_dpdk_releases() + format_output = FormatOutput(args.format_output, dpdk_releases) + + map_grammar = MAP_GRAMMAR.format(get_abi_versions()) + map_parser = makeGrammar(map_grammar, {}) + + format_output.print_columns() + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + relsym = [str(path)] + + for release in dpdk_releases: + csym = count_release_symbols(map_parser, release, path) + relsym += csym + + format_output.print_row(relsym) + +if __name__ == '__main__': + main() -- 2.26.2
Script to track growth of stable and experimental symbols over releases since v19.11. Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- v2: reworked to fix pylint errors v3: sent with the correct in-reply-to v4: fix typos picked up by the CI devtools/count_symbols.py | 262 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100755 devtools/count_symbols.py diff --git a/devtools/count_symbols.py b/devtools/count_symbols.py new file mode 100755 index 0000000000..6194df0318 --- /dev/null +++ b/devtools/count_symbols.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +'''Tool to count the number of symbols in each DPDK release''' +from pathlib import Path +import sys +import os +import subprocess +import argparse +import re +import datetime + +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + sys.exit() + +MAP_GRAMMAR = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" + +def get_abi_versions(): + '''Returns a string of possible dpdk abi versions''' + + year = datetime.date.today().year - 2000 + tags = " |".join(['\'{}\''.format(i) \ + for i in reversed(range(21, year + 1)) ]) + tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return tags + +def get_dpdk_releases(): + '''Returns a list of dpdk release tags names since v19.11''' + + year = datetime.date.today().year - 2000 + year_range = "|".join("{}".format(i) for i in range(19,year + 1)) + pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + try: + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + print("Failed to interogate git for release tags") + sys.exit() + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [ tag.replace('\"','') \ + for tag in reversed(tags) \ + if pattern.match(tag) ][:-3] + + return tags + +def fix_directory_name(path): + '''Prepend librte to the source directory name''' + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + +def directory_renamed(path, rel): + '''Fix removal of the librte_ from the directory names''' + + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + return result + +def mapfile_renamed(path, rel): + '''Fix renaming of the map file''' + newfile = None + + result = subprocess.run(['git', 'ls-tree', \ + rel, str(path.parent) + '/'], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if newfile is not None: + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + else: + result = None + + return result + +def mapfile_and_directory_renamed(path, rel): + '''Fix renaming of the map file & the source directory''' + mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name)) + + return mapfile_renamed(mapfilepath, rel) + +def get_terminal_rows(): + '''Find the number of rows in the terminal''' + + rows, _ = os.popen('stty size', 'r').read().split() + return int(rows) + +class FormatOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.OUTPUT_FORMATS[format_output](self,dpdk_releases) + self.column_titles = ['mapfile'] + dpdk_releases + + self.terminal_rows = get_terminal_rows() + self.row = 0 + + def set_terminal_output(self,dpdk_rel): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}' + \ + ''.join(['{:<6}{:<6}'] * (len(dpdk_rel))) + self.column_fmt = '{:50}' + \ + ''.join(['{:<12}'] * (len(dpdk_rel))) + + def set_csv_output(self,dpdk_rel): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},' + \ + ','.join(['{},{}'] * (len(dpdk_rel))) + self.column_fmt = '{},' + \ + ','.join(['{},'] * (len(dpdk_rel))) + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + self.row += 1 + + def print_row(self,symbols): + '''Print row of symbol values''' + print(self.output_fmt.format(*symbols)) + self.row += 1 + + if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)): + self.print_columns() + + OUTPUT_FORMATS = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } + +SRC_DIRECTORIES = 'drivers, lib' +IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL'] +FIX_STRATEGIES = [directory_renamed, \ + mapfile_renamed, \ + mapfile_and_directory_renamed] + +def count_release_symbols(map_parser, release, mapfile_path): + '''Count the symbols for a given release and mapfile''' + csym = [0] * 2 + abi_sections = None + + tagfile = '{}:{}'.format(release,mapfile_path) + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + for fix_strategy in FIX_STRATEGIES: + if result is not None: + break + result = fix_strategy(mapfile_path, release) + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = map_parser(mapfile).abi() + + if abi_sections is not None: + # which versions are present, and we care about + found_ver = [ver \ + for ver in abi_sections \ + if ver not in IGNORE_SECTIONS] + + for ver in found_ver: + csym[0] += len(abi_sections[ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in abi_sections: + csym[1] = len(abi_sections['EXPERIMENTAL']) + + return csym + +def main(): + '''Main entry point''' + + parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs') + parser.add_argument('--format-output', choices=['terminal','csv'], \ + default='terminal') + parser.add_argument('--directory', choices=SRC_DIRECTORIES, + default=SRC_DIRECTORIES) + args = parser.parse_args() + + dpdk_releases = get_dpdk_releases() + format_output = FormatOutput(args.format_output, dpdk_releases) + + map_grammar = MAP_GRAMMAR.format(get_abi_versions()) + map_parser = makeGrammar(map_grammar, {}) + + format_output.print_columns() + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + relsym = [str(path)] + + for release in dpdk_releases: + csym = count_release_symbols(map_parser, release, path) + relsym += csym + + format_output.print_row(relsym) + +if __name__ == '__main__': + main() -- 2.26.2
On Mon, 21 Jun 2021 16:35:31 +0100
Ray Kinsella <mdr@ashroe.eu> wrote:
> +def get_terminal_rows():
> + '''Find the number of rows in the terminal'''
> +
> + rows, _ = os.popen('stty size', 'r').read().split()
> + return int(rows)
> +
Use standard Python function os.get_terminal_size or shutil.get_terminal_size() instead.
That way it would be portable to other OS.
Script to track growth of stable and experimental symbols over releases since v19.11. Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- v2: reworked to fix pylint errors v3: sent with the correct in-reply-to v4: fix typos picked up by the CI v5: fix terminal_size & directory args devtools/count_symbols.py | 262 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100755 devtools/count_symbols.py diff --git a/devtools/count_symbols.py b/devtools/count_symbols.py new file mode 100755 index 0000000000..96990f609f --- /dev/null +++ b/devtools/count_symbols.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +'''Tool to count the number of symbols in each DPDK release''' +from pathlib import Path +import sys +import os +import subprocess +import argparse +import re +import datetime + +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + sys.exit() + +MAP_GRAMMAR = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" + +def get_abi_versions(): + '''Returns a string of possible dpdk abi versions''' + + year = datetime.date.today().year - 2000 + tags = " |".join(['\'{}\''.format(i) \ + for i in reversed(range(21, year + 1)) ]) + tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return tags + +def get_dpdk_releases(): + '''Returns a list of dpdk release tags names since v19.11''' + + year = datetime.date.today().year - 2000 + year_range = "|".join("{}".format(i) for i in range(19,year + 1)) + pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + try: + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + print("Failed to interogate git for release tags") + sys.exit() + + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [ tag.replace('\"','') \ + for tag in reversed(tags) \ + if pattern.match(tag) ][:-3] + + return tags + +def fix_directory_name(path): + '''Prepend librte to the source directory name''' + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + +def directory_renamed(path, rel): + '''Fix removal of the librte_ from the directory names''' + + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + return result + +def mapfile_renamed(path, rel): + '''Fix renaming of the map file''' + newfile = None + + result = subprocess.run(['git', 'ls-tree', \ + rel, str(path.parent) + '/'], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if newfile is not None: + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + else: + result = None + + return result + +def mapfile_and_directory_renamed(path, rel): + '''Fix renaming of the map file & the source directory''' + mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name)) + + return mapfile_renamed(mapfilepath, rel) + +def get_terminal_rows(): + '''Find the number of rows in the terminal''' + + return os.get_terminal_size().lines + +class FormatOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.OUTPUT_FORMATS[format_output](self,dpdk_releases) + self.column_titles = ['mapfile'] + dpdk_releases + + self.terminal_rows = get_terminal_rows() + self.row = 0 + + def set_terminal_output(self,dpdk_rel): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}' + \ + ''.join(['{:<6}{:<6}'] * (len(dpdk_rel))) + self.column_fmt = '{:50}' + \ + ''.join(['{:<12}'] * (len(dpdk_rel))) + + def set_csv_output(self,dpdk_rel): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},' + \ + ','.join(['{},{}'] * (len(dpdk_rel))) + self.column_fmt = '{},' + \ + ','.join(['{},'] * (len(dpdk_rel))) + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + self.row += 1 + + def print_row(self,symbols): + '''Print row of symbol values''' + print(self.output_fmt.format(*symbols)) + self.row += 1 + + if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)): + self.print_columns() + + OUTPUT_FORMATS = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } + +SRC_DIRECTORIES = 'drivers,lib' +IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL'] +FIX_STRATEGIES = [directory_renamed, \ + mapfile_renamed, \ + mapfile_and_directory_renamed] + +def count_release_symbols(map_parser, release, mapfile_path): + '''Count the symbols for a given release and mapfile''' + csym = [0] * 2 + abi_sections = None + + tagfile = '{}:{}'.format(release,mapfile_path) + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + for fix_strategy in FIX_STRATEGIES: + if result is not None: + break + result = fix_strategy(mapfile_path, release) + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = map_parser(mapfile).abi() + + if abi_sections is not None: + # which versions are present, and we care about + found_ver = [ver \ + for ver in abi_sections \ + if ver not in IGNORE_SECTIONS] + + for ver in found_ver: + csym[0] += len(abi_sections[ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in abi_sections: + csym[1] = len(abi_sections['EXPERIMENTAL']) + + return csym + +def main(): + '''Main entry point''' + + parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs') + parser.add_argument('--format-output', choices=['terminal','csv'], \ + default='terminal') + parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','), + default=SRC_DIRECTORIES) + args = parser.parse_args() + + dpdk_releases = get_dpdk_releases() + format_output = FormatOutput(args.format_output, dpdk_releases) + + map_grammar = MAP_GRAMMAR.format(get_abi_versions()) + map_parser = makeGrammar(map_grammar, {}) + + format_output.print_columns() + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + relsym = [str(path)] + + for release in dpdk_releases: + csym = count_release_symbols(map_parser, release, path) + relsym += csym + + format_output.print_row(relsym) + +if __name__ == '__main__': + main() -- 2.26.2
This script tracks the growth of stable and experimental symbols over releases since v19.11. The script has the ability to count the added symbols between two dpdk releases, and to list experimental symbols present in two dpdk releases (expired symbols). example usages: Count symbols added since v19.11 $ devtools/symbol_tool.py count-symbols Count symbols added since v20.11 $ devtools/symbol_tool.py count-symbols --releases v20.11,v21.05 List experimental symbols present in v20.11 and v21.05 $ devtools/symbol_tool.py list-expired --releases v20.11,v21.05 List experimental symbols in libraries only, present since v19.11 $ devtools/symbol_tool.py list-expired --directory lib Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- v2: reworked to fix pylint errors v3: sent with the correct in-reply-to v4: fix typos picked up by the CI v5: fix terminal_size & directory args v6: added list-expired, to list expired experimental symbols devtools/symbol_tool.py | 377 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 377 insertions(+) create mode 100755 devtools/symbol_tool.py diff --git a/devtools/symbol_tool.py b/devtools/symbol_tool.py new file mode 100755 index 0000000000..63969a131b --- /dev/null +++ b/devtools/symbol_tool.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +'''Tool to count or list symbols in each DPDK release''' +from pathlib import Path +import sys +import os +import subprocess +import argparse +import re +import datetime +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + sys.exit() + +MAP_GRAMMAR = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" + +def get_abi_versions(): + '''Returns a string of possible dpdk abi versions''' + + year = datetime.date.today().year - 2000 + tags = " |".join(['\'{}\''.format(i) \ + for i in reversed(range(21, year + 1)) ]) + tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return tags + +def get_dpdk_releases(): + '''Returns a list of dpdk release tags names since v19.11''' + + year = datetime.date.today().year - 2000 + year_range = "|".join("{}".format(i) for i in range(19,year + 1)) + pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + try: + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + print("Failed to interogate git for release tags") + sys.exit() + + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [ tag.replace('\"','') \ + for tag in reversed(tags) \ + if pattern.match(tag) ][:-3] + + return tags + +def fix_directory_name(path): + '''Prepend librte to the source directory name''' + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + +def directory_renamed(path, rel): + '''Fix removal of the librte_ from the directory names''' + + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + return result + +def mapfile_renamed(path, rel): + '''Fix renaming of the map file''' + newfile = None + + result = subprocess.run(['git', 'ls-tree', \ + rel, str(path.parent) + '/'], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if newfile is not None: + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + else: + result = None + + return result + +def mapfile_and_directory_renamed(path, rel): + '''Fix renaming of the map file & the source directory''' + mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name)) + + return mapfile_renamed(mapfilepath, rel) + +FIX_STRATEGIES = [directory_renamed, \ + mapfile_renamed, \ + mapfile_and_directory_renamed] + +def get_symbols(map_parser, release, mapfile_path): + '''Count the symbols for a given release and mapfile''' + abi_sections = {} + + tagfile = '{}:{}'.format(release,mapfile_path) + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + for fix_strategy in FIX_STRATEGIES: + if result is not None: + break + result = fix_strategy(mapfile_path, release) + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = map_parser(mapfile).abi() + + return abi_sections + +def get_terminal_rows(): + '''Find the number of rows in the terminal''' + + try: + return os.get_terminal_size().lines + except IOError: + return 0 + +class SymbolCountOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.OUTPUT_FORMATS[format_output](self,dpdk_releases) + self.column_titles = ['mapfile'] + dpdk_releases + + self.terminal_rows = get_terminal_rows() + self.row = 0 + + def set_terminal_output(self,dpdk_rel): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}' + \ + ''.join(['{:<6}{:<6}'] * (len(dpdk_rel))) + self.column_fmt = '{:50}' + \ + ''.join(['{:<12}'] * (len(dpdk_rel))) + + def set_csv_output(self,dpdk_rel): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},' + \ + ','.join(['{},{}'] * (len(dpdk_rel))) + self.column_fmt = '{},' + \ + ','.join(['{},'] * (len(dpdk_rel))) + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + self.row += 1 + + def print_row(self, mapfile, symbols): + '''Print row of symbol values''' + print(self.output_fmt.format(*([mapfile] + symbols))) + self.row += 1 + + if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)): + self.print_columns() + + OUTPUT_FORMATS = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } + +class ListExpiredOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.terminal = True + self.OUTPUT_FORMATS[format_output](self,dpdk_releases) + self.column_titles = ['mapfile'] + \ + ['expired (' + ','.join(dpdk_releases) + ')'] + + def set_terminal_output(self, _): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}{:<50}' + self.column_fmt = '{:50}{:50}' + + def set_csv_output(self, _): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},{}' + self.column_fmt = '{},{}' + self.terminal = False + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + + def print_row(self, mapfile, symbols): + '''Print row of symbol values''' + + for symbol in symbols: + print(self.output_fmt.format(mapfile,symbol)) + if self.terminal : + mapfile = '' + + OUTPUT_FORMATS = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } + +class CountSymbolsAction: + ''' Logic to count symbols added since a give release ''' + IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL'] + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.symbols_count = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbol_count = experimental_count = 0 + + symbols = get_symbols(self.parser, release, self.path) + + # which versions are present, and we care about + abi_vers = [abi_ver \ + for abi_ver in symbols \ + if abi_ver not in self.IGNORE_SECTIONS] + + for abi_ver in abi_vers: + symbol_count += len(symbols[abi_ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in symbols.keys(): + experimental_count = len(symbols['EXPERIMENTAL']) + + self.symbols_count += [symbol_count, experimental_count] + + def __del__(self): + self.format_output.print_row(self.path.parent.name, self.symbols_count) + +class ListExpiredAction: + ''' Logic to list expired symbols between two releases ''' + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.experimental_symbols = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbols = get_symbols(self.parser, release, self.path) + if 'EXPERIMENTAL' in symbols.keys(): + self.experimental_symbols.append(symbols['EXPERIMENTAL']) + + def __del__(self): + if len(self.experimental_symbols) != 2: + return + + tmp = self.experimental_symbols + # find symbols present in both dpdk releases + intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]] + + # check for empty set + if intersect_syms == []: + return + + self.format_output.print_row(self.path.parent.name, intersect_syms) + +SRC_DIRECTORIES = 'drivers,lib' + +ACTIONS = {None: CountSymbolsAction, \ + 'count-symbols': CountSymbolsAction, \ + 'list-expired': ListExpiredAction} + +ACTION_OUTPUT = {None: SymbolCountOutput, \ + 'count-symbols': SymbolCountOutput, \ + 'list-expired': ListExpiredOutput} + +def main(): + '''Main entry point''' + + dpdk_releases = get_dpdk_releases() + + parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs') + parser.add_argument('mode', choices=['count-symbols','list-expired']) + parser.add_argument('--format-output', choices=['terminal','csv'], \ + default='terminal') + parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','), + default=SRC_DIRECTORIES) + parser.add_argument('--releases', \ + help='2 x comma seperated release tags e.g. \'' \ + + ','.join([dpdk_releases[0],dpdk_releases[-1]]) \ + + '\'') + args = parser.parse_args() + + if args.releases is not None: + dpdk_releases = args.releases.split(',') + + if args.mode == 'list-expired': + if len(dpdk_releases) < 2: + sys.exit('Please specify two releases to compare ' \ + 'in \'list-expired\' mode.') + dpdk_releases = [dpdk_releases[0], dpdk_releases[len(dpdk_releases) - 1]] + + action = ACTIONS[args.mode] + format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases) + + map_grammar = MAP_GRAMMAR.format(get_abi_versions()) + map_parser = makeGrammar(map_grammar, {}) + + format_output.print_columns() + + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + release_action = action(path, map_parser, format_output) + + for release in dpdk_releases: + release_action.add_mapfile(release) + + # all the magic happens in the destructor + del release_action + +if __name__ == '__main__': + main() -- 2.26.2
This script tracks the growth of stable and experimental symbols over releases since v19.11. The script has the ability to count the added symbols between two dpdk releases, and to list experimental symbols present in two dpdk releases (expired symbols). example usages: Count symbols added since v19.11 $ devtools/symbol_tool.py count-symbols Count symbols added since v20.11 $ devtools/symbol_tool.py count-symbols --releases v20.11,v21.05 List experimental symbols present in v20.11 and v21.05 $ devtools/symbol_tool.py list-expired --releases v20.11,v21.05 List experimental symbols in libraries only, present since v19.11 $ devtools/symbol_tool.py list-expired --directory lib Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- v2: reworked to fix pylint errors v3: sent with the correct in-reply-to v4: fix typos picked up by the CI v5: fix terminal_size & directory args v6: added list-expired, to list expired experimental symbols v7: fix typo in comments devtools/symbol_tool.py | 377 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 377 insertions(+) create mode 100755 devtools/symbol_tool.py diff --git a/devtools/symbol_tool.py b/devtools/symbol_tool.py new file mode 100755 index 0000000000..f2a2d43a15 --- /dev/null +++ b/devtools/symbol_tool.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +'''Tool to count or list symbols in each DPDK release''' +from pathlib import Path +import sys +import os +import subprocess +import argparse +import re +import datetime +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + sys.exit() + +MAP_GRAMMAR = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" + +def get_abi_versions(): + '''Returns a string of possible dpdk abi versions''' + + year = datetime.date.today().year - 2000 + tags = " |".join(['\'{}\''.format(i) \ + for i in reversed(range(21, year + 1)) ]) + tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return tags + +def get_dpdk_releases(): + '''Returns a list of dpdk release tags names since v19.11''' + + year = datetime.date.today().year - 2000 + year_range = "|".join("{}".format(i) for i in range(19,year + 1)) + pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + try: + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + print("Failed to interogate git for release tags") + sys.exit() + + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [ tag.replace('\"','') \ + for tag in reversed(tags) \ + if pattern.match(tag) ][:-3] + + return tags + +def fix_directory_name(path): + '''Prepend librte to the source directory name''' + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + +def directory_renamed(path, rel): + '''Fix removal of the librte_ from the directory names''' + + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + return result + +def mapfile_renamed(path, rel): + '''Fix renaming of the map file''' + newfile = None + + result = subprocess.run(['git', 'ls-tree', \ + rel, str(path.parent) + '/'], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if newfile is not None: + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + else: + result = None + + return result + +def mapfile_and_directory_renamed(path, rel): + '''Fix renaming of the map file & the source directory''' + mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name)) + + return mapfile_renamed(mapfilepath, rel) + +FIX_STRATEGIES = [directory_renamed, \ + mapfile_renamed, \ + mapfile_and_directory_renamed] + +def get_symbols(map_parser, release, mapfile_path): + '''Count the symbols for a given release and mapfile''' + abi_sections = {} + + tagfile = '{}:{}'.format(release,mapfile_path) + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + for fix_strategy in FIX_STRATEGIES: + if result is not None: + break + result = fix_strategy(mapfile_path, release) + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = map_parser(mapfile).abi() + + return abi_sections + +def get_terminal_rows(): + '''Find the number of rows in the terminal''' + + try: + return os.get_terminal_size().lines + except IOError: + return 0 + +class SymbolCountOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.OUTPUT_FORMATS[format_output](self,dpdk_releases) + self.column_titles = ['mapfile'] + dpdk_releases + + self.terminal_rows = get_terminal_rows() + self.row = 0 + + def set_terminal_output(self,dpdk_rel): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}' + \ + ''.join(['{:<6}{:<6}'] * (len(dpdk_rel))) + self.column_fmt = '{:50}' + \ + ''.join(['{:<12}'] * (len(dpdk_rel))) + + def set_csv_output(self,dpdk_rel): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},' + \ + ','.join(['{},{}'] * (len(dpdk_rel))) + self.column_fmt = '{},' + \ + ','.join(['{},'] * (len(dpdk_rel))) + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + self.row += 1 + + def print_row(self, mapfile, symbols): + '''Print row of symbol values''' + print(self.output_fmt.format(*([mapfile] + symbols))) + self.row += 1 + + if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)): + self.print_columns() + + OUTPUT_FORMATS = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } + +class ListExpiredOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.terminal = True + self.OUTPUT_FORMATS[format_output](self,dpdk_releases) + self.column_titles = ['mapfile'] + \ + ['expired (' + ','.join(dpdk_releases) + ')'] + + def set_terminal_output(self, _): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}{:<50}' + self.column_fmt = '{:50}{:50}' + + def set_csv_output(self, _): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},{}' + self.column_fmt = '{},{}' + self.terminal = False + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + + def print_row(self, mapfile, symbols): + '''Print row of symbol values''' + + for symbol in symbols: + print(self.output_fmt.format(mapfile,symbol)) + if self.terminal : + mapfile = '' + + OUTPUT_FORMATS = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } + +class CountSymbolsAction: + ''' Logic to count symbols added since a give release ''' + IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL'] + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.symbols_count = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbol_count = experimental_count = 0 + + symbols = get_symbols(self.parser, release, self.path) + + # which versions are present, and we care about + abi_vers = [abi_ver \ + for abi_ver in symbols \ + if abi_ver not in self.IGNORE_SECTIONS] + + for abi_ver in abi_vers: + symbol_count += len(symbols[abi_ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in symbols.keys(): + experimental_count = len(symbols['EXPERIMENTAL']) + + self.symbols_count += [symbol_count, experimental_count] + + def __del__(self): + self.format_output.print_row(self.path.parent.name, self.symbols_count) + +class ListExpiredAction: + ''' Logic to list expired symbols between two releases ''' + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.experimental_symbols = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbols = get_symbols(self.parser, release, self.path) + if 'EXPERIMENTAL' in symbols.keys(): + self.experimental_symbols.append(symbols['EXPERIMENTAL']) + + def __del__(self): + if len(self.experimental_symbols) != 2: + return + + tmp = self.experimental_symbols + # find symbols present in both dpdk releases + intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]] + + # check for empty set + if intersect_syms == []: + return + + self.format_output.print_row(self.path.parent.name, intersect_syms) + +SRC_DIRECTORIES = 'drivers,lib' + +ACTIONS = {None: CountSymbolsAction, \ + 'count-symbols': CountSymbolsAction, \ + 'list-expired': ListExpiredAction} + +ACTION_OUTPUT = {None: SymbolCountOutput, \ + 'count-symbols': SymbolCountOutput, \ + 'list-expired': ListExpiredOutput} + +def main(): + '''Main entry point''' + + dpdk_releases = get_dpdk_releases() + + parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs') + parser.add_argument('mode', choices=['count-symbols','list-expired']) + parser.add_argument('--format-output', choices=['terminal','csv'], \ + default='terminal') + parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','), + default=SRC_DIRECTORIES) + parser.add_argument('--releases', \ + help='2 x comma separated release tags e.g. \'' \ + + ','.join([dpdk_releases[0],dpdk_releases[-1]]) \ + + '\'') + args = parser.parse_args() + + if args.releases is not None: + dpdk_releases = args.releases.split(',') + + if args.mode == 'list-expired': + if len(dpdk_releases) < 2: + sys.exit('Please specify two releases to compare ' \ + 'in \'list-expired\' mode.') + dpdk_releases = [dpdk_releases[0], dpdk_releases[len(dpdk_releases) - 1]] + + action = ACTIONS[args.mode] + format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases) + + map_grammar = MAP_GRAMMAR.format(get_abi_versions()) + map_parser = makeGrammar(map_grammar, {}) + + format_output.print_columns() + + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + release_action = action(path, map_parser, format_output) + + for release in dpdk_releases: + release_action.add_mapfile(release) + + # all the magic happens in the destructor + del release_action + +if __name__ == '__main__': + main() -- 2.26.2
Scripts to count and track the lifecycle of DPDK symbols. The symbol-tool script reports on the growth of symbols over releases and list expired symbols. The notify-symbol-maintainers script consumes the input from symbol-tool and generates email notifications of expired symbols. v2: reworked to fix pylint errors v3: sent with the correct in-reply-to v4: fix typos picked up by the CI v5: fix terminal_size & directory args v6: added list-expired, to list expired experimental symbols v7: fix typo in comments v8: added tool to notify maintainers of expired symbols Ray Kinsella (2): devtools: script to track map symbols devtools: script to send notifications of expired symbols devtools/notify-symbol-maintainers.py | 224 ++++++++++++++ devtools/symbol-tool.py | 402 ++++++++++++++++++++++++++ 2 files changed, 626 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py create mode 100755 devtools/symbol-tool.py -- 2.26.2
This script tracks the growth of stable and experimental symbols over releases since v19.11. The script has the ability to count the added symbols between two dpdk releases, and to list experimental symbols present in two dpdk releases (expired symbols). example usages: Count symbols added since v19.11 $ devtools/symbol-tool.py count-symbols Count symbols added since v20.11 $ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05 List experimental symbols present in v20.11 and v21.05 $ devtools/symbol-tool.py list-expired --releases v20.11,v21.05 List experimental symbols in libraries only, present since v19.11 $ devtools/symbol-tool.py list-expired --directory lib Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/symbol-tool.py | 402 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 402 insertions(+) create mode 100755 devtools/symbol-tool.py diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py new file mode 100755 index 0000000000..39727c9a32 --- /dev/null +++ b/devtools/symbol-tool.py @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +'''Tool to count or list symbols in each DPDK release''' +from pathlib import Path +import sys +import os +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import re +import datetime +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + sys.exit() + +DESCRIPTION = ''' +This script tracks the growth of stable and experimental symbols +over releases since v19.11. The script has the ability to +count the added symbols between two dpdk releases, and to +list experimental symbols present in two dpdk releases +(expired symbols). + +example usages: + +Count symbols added since v19.11 +$ devtools/symbol-tool.py count-symbols + +Count symbols added since v20.11 +$ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05 + +List experimental symbols present in v20.11 and v21.05 +$ devtools/symbol-tool.py list-expired --releases v20.11,v21.05 + +List experimental symbols in libraries only, present since v19.11 +$ devtools/symbol-tool.py list-expired --directory lib +''' + +MAP_GRAMMAR = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" + +def get_abi_versions(): + '''Returns a string of possible dpdk abi versions''' + + year = datetime.date.today().year - 2000 + tags = " |".join(['\'{}\''.format(i) \ + for i in reversed(range(21, year + 1)) ]) + tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return tags + +def get_dpdk_releases(): + '''Returns a list of dpdk release tags names since v19.11''' + + year = datetime.date.today().year - 2000 + year_range = "|".join("{}".format(i) for i in range(19,year + 1)) + pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + try: + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + print("Failed to interogate git for release tags") + sys.exit() + + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [ tag.replace('\"','') \ + for tag in reversed(tags) \ + if pattern.match(tag) ][:-3] + + return tags + +def fix_directory_name(path): + '''Prepend librte to the source directory name''' + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + +def directory_renamed(path, rel): + '''Fix removal of the librte_ from the directory names''' + + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + return result + +def mapfile_renamed(path, rel): + '''Fix renaming of the map file''' + newfile = None + + result = subprocess.run(['git', 'ls-tree', \ + rel, str(path.parent) + '/'], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if newfile is not None: + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + else: + result = None + + return result + +def mapfile_and_directory_renamed(path, rel): + '''Fix renaming of the map file & the source directory''' + mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name)) + + return mapfile_renamed(mapfilepath, rel) + +FIX_STRATEGIES = [directory_renamed, \ + mapfile_renamed, \ + mapfile_and_directory_renamed] + +def get_symbols(map_parser, release, mapfile_path): + '''Count the symbols for a given release and mapfile''' + abi_sections = {} + + tagfile = '{}:{}'.format(release,mapfile_path) + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + for fix_strategy in FIX_STRATEGIES: + if result is not None: + break + result = fix_strategy(mapfile_path, release) + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = map_parser(mapfile).abi() + + return abi_sections + +def get_terminal_rows(): + '''Find the number of rows in the terminal''' + + try: + return os.get_terminal_size().lines + except IOError: + return 0 + +class SymbolCountOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.OUTPUT_FORMATS[format_output](self,dpdk_releases) + self.column_titles = ['mapfile'] + dpdk_releases + + self.terminal_rows = get_terminal_rows() + self.row = 0 + + def set_terminal_output(self,dpdk_rel): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}' + \ + ''.join(['{:<6}{:<6}'] * (len(dpdk_rel))) + self.column_fmt = '{:50}' + \ + ''.join(['{:<12}'] * (len(dpdk_rel))) + + def set_csv_output(self,dpdk_rel): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},' + \ + ','.join(['{},{}'] * (len(dpdk_rel))) + self.column_fmt = '{},' + \ + ','.join(['{},'] * (len(dpdk_rel))) + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + self.row += 1 + + def print_row(self, mapfile, symbols): + '''Print row of symbol values''' + print(self.output_fmt.format(*([mapfile] + symbols))) + self.row += 1 + + if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)): + self.print_columns() + + OUTPUT_FORMATS = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } + +class ListExpiredOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.terminal = True + self.OUTPUT_FORMATS[format_output](self,dpdk_releases) + self.column_titles = ['mapfile'] + \ + ['expired (' + ','.join(dpdk_releases) + ')'] + + def set_terminal_output(self, _): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}{:<50}' + self.column_fmt = '{:50}{:50}' + + def set_csv_output(self, _): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},{}' + self.column_fmt = '{},{}' + self.terminal = False + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + + def print_row(self, mapfile, symbols): + '''Print row of symbol values''' + + for symbol in symbols: + print(self.output_fmt.format(mapfile,symbol)) + if self.terminal : + mapfile = '' + + OUTPUT_FORMATS = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } + +class CountSymbolsAction: + ''' Logic to count symbols added since a give release ''' + IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL'] + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.symbols_count = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbol_count = experimental_count = 0 + + symbols = get_symbols(self.parser, release, self.path) + + # which versions are present, and we care about + abi_vers = [abi_ver \ + for abi_ver in symbols \ + if abi_ver not in self.IGNORE_SECTIONS] + + for abi_ver in abi_vers: + symbol_count += len(symbols[abi_ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in symbols.keys(): + experimental_count = len(symbols['EXPERIMENTAL']) + + self.symbols_count += [symbol_count, experimental_count] + + def __del__(self): + self.format_output.print_row(self.path.parent, self.symbols_count) + +class ListExpiredAction: + ''' Logic to list expired symbols between two releases ''' + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.experimental_symbols = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbols = get_symbols(self.parser, release, self.path) + if 'EXPERIMENTAL' in symbols.keys(): + self.experimental_symbols.append(symbols['EXPERIMENTAL']) + + def __del__(self): + if len(self.experimental_symbols) != 2: + return + + tmp = self.experimental_symbols + # find symbols present in both dpdk releases + intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]] + + # check for empty set + if intersect_syms == []: + return + + self.format_output.print_row(self.path.parent, intersect_syms) + +SRC_DIRECTORIES = 'drivers,lib' + +ACTIONS = {None: CountSymbolsAction, \ + 'count-symbols': CountSymbolsAction, \ + 'list-expired': ListExpiredAction} + +ACTION_OUTPUT = {None: SymbolCountOutput, \ + 'count-symbols': SymbolCountOutput, \ + 'list-expired': ListExpiredOutput} + +def main(): + '''Main entry point''' + + dpdk_releases = get_dpdk_releases() + + parser = argparse.ArgumentParser(description=DESCRIPTION, \ + formatter_class=RawTextHelpFormatter + ) + parser.add_argument('mode', choices=['count-symbols','list-expired']) + parser.add_argument('--format-output', choices=['terminal','csv'], \ + default='terminal') + parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','), + default=SRC_DIRECTORIES) + parser.add_argument('--releases', \ + help='2 x comma separated release tags e.g. \'' \ + + ','.join([dpdk_releases[0],dpdk_releases[-1]]) \ + + '\'') + args = parser.parse_args() + + if args.releases is not None: + dpdk_releases = args.releases.split(',') + + if args.mode == 'list-expired': + if len(dpdk_releases) < 2: + sys.exit('Please specify two releases to compare ' \ + 'in \'list-expired\' mode.') + dpdk_releases = [dpdk_releases[0], dpdk_releases[len(dpdk_releases) - 1]] + + action = ACTIONS[args.mode] + format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases) + + map_grammar = MAP_GRAMMAR.format(get_abi_versions()) + map_parser = makeGrammar(map_grammar, {}) + + format_output.print_columns() + + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + release_action = action(path, map_parser, format_output) + + for release in dpdk_releases: + release_action.add_mapfile(release) + + # all the magic happens in the destructor + del release_action + +if __name__ == '__main__': + main() -- 2.26.2
Use this script with the output of the DPDK symbol tool, to notify maintainers of expired symbols by email. You need to define the environment variable DPDK_GETMAINTAINER_PATH, for this tool to work. Use terminal output to review the emails before sending. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ devtools/notify_expired_symbols.py --format-output terminal Then use email output to send the emails to the maintainers. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ --smtp-server <server> --sender <someone@somewhere.com> \ --password <password> Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/notify-symbol-maintainers.py | 224 ++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py new file mode 100755 index 0000000000..447f88bb03 --- /dev/null +++ b/devtools/notify-symbol-maintainers.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +'''Tool to notify maintainers of expired symbols''' +import smtplib +import ssl +import sys +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import time +from email.message import EmailMessage + +DESCRIPTION = ''' +Use this script with the output of the DPDK symbol tool, to notify maintainers +of expired symbols by email. You need to define the environment variable +DPDK_GETMAINTAINER_PATH, for this tool to work. + +Use terminal output to review the emails before sending. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +devtools/notify_expired_symbols.py --format-output terminal + +Then use email output to send the emails to the maintainers. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +--smtp-server <server> --sender <someone@somewhere.com> --password <password> +''' + +EMAIL_TEMPLATE = '''Hi there, + +Please note the symbols listed below have expired. In line with the DPDK ABI +policy, they should be scheduled for removal, in the next DPDK release. + +For more information, please see the DPDK ABI Policy, section 3.5.3. +https://doc.dpdk.org/guides/contributing/abi_policy.html + +Thanks, + +The DPDK Symbol Bot + +''' + +default_maintainers = ['Ray Kinsella <mdr@ashroe.eu>', \ + 'Thomas Monjalon <thomas@monjalon.net>'] +get_maintainer = ['devtools/get-maintainer.sh', \ + '--email', '-f'] + +def get_maintainers(libpath): + '''Get the maintainers for given library''' + try: + cmd = get_maintainer + [libpath] + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + if result is not None: + email = result.stdout.decode('utf-8') + if email == '': + email = default_maintainers + else: + email = list(filter(None,email.split('\n'))) + else: + email = default_maintainers + + return email + +def get_message(library, symbols): + '''Build email message from symbols, config and maintainers''' + message = {} + maintainers = get_maintainers(library) + + message['To'] = maintainers + if maintainers != default_maintainers: + message['CC'] = default_maintainers + + message['Subject'] = 'Expired symbols in {}\n'.format(library) + + body = EMAIL_TEMPLATE + for sym in symbols: + body += ('{}\n'.format(sym)) + + message['Body'] = body + + return message + +class OutputEmail(): + '''Format the output for email''' + def __init__(self, config): + self.config = config + + self.terminal = OutputTerminal(config) + context = ssl.create_default_context() + + # Try to log in to server and send email + try: + self.server = smtplib.SMTP(config['smtp_server'], 587) + self.server.starttls(context=context) # Secure the connection + self.server.login(config['sender'], config['password']) + except Exception as exception: + print(exception) + raise exception + + def message(self,message): + '''send email''' + self.terminal.message(message) + + msg = EmailMessage() + msg.set_content(message.pop('Body')) + + for key in message.keys(): + msg[key] = message[key] + + msg['From'] = self.config['sender'] + msg['Reply-To'] = 'no-reply@dpdk.org' + + self.server.send_message(msg) + + time.sleep(1) + + def __del__(self): + self.server.quit() + +class OutputTerminal(): # pylint: disable=too-few-public-methods + '''Format the output for the terminal''' + def __init__(self, config): + self.config = config + + def message(self,message): + '''Print email to terminal''' + terminal = 'To:' + ', '.join(message['To']) + '\n' + if 'sender' in self.config.keys(): + terminal += 'From:' + self.config['sender'] + '\n' + + terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n' + if 'CC' in message.keys(): + terminal += 'CC:' + ', '.join(message['CC']) + '\n' + + terminal += 'Subject:' + message['Subject'] + '\n' + terminal += 'Body:' + message['Body'] + '\n' + + print(terminal) + print('-' * 80) + +def parse_config(args): + '''put the command line args in the right places''' + config = {} + error_msg = None + + outputs = { + None : OutputTerminal, + 'terminal' : OutputTerminal, + 'email' : OutputEmail + } + + if args.format_output == 'email': + if args.smtp_server is None: + error_msg = 'SMTP server' + else: + config['smtp_server'] = args.smtp_server + + if args.sender is None: + error_msg = 'sender' + else: + config['sender'] = args.sender + + if args.password is None: + error_msg = 'password' + else: + config['password'] = args.password + + if error_msg is not None: + print('Please specify a {} for email output'.format(error_msg)) + return None + + config['output'] = outputs[args.format_output] + return config + +def main(): + '''Main entry point''' + parser = argparse.ArgumentParser(description=DESCRIPTION, \ + formatter_class=RawTextHelpFormatter) + parser.add_argument('--format-output', choices=['terminal','email'], \ + default='terminal') + parser.add_argument('--smtp-server') + parser.add_argument('--password') + parser.add_argument('--sender') + + args = parser.parse_args() + config = parse_config(args) + if config is None: + return + + symbols = [] + lastlib = library = '' + + output = config['output'](config) + + for line in sys.stdin: + line = line.rstrip('\n') + library, symbol = [line[:line.find(',')], \ + line[line.find(',') + 1: len(line)]] + if library == 'mapfile': + continue + + if library != lastlib: + message = get_message(lastlib, symbols) + output.message(message) + symbols = [] + + lastlib = library + symbols = symbols + [symbol] + + #print the last library + message = get_message(lastlib, symbols) + output.message(message) + +if __name__ == '__main__': + main() -- 2.26.2
Scripts to count and track the lifecycle of DPDK symbols. The symbol-tool script reports on the growth of symbols over releases and list expired symbols. The notify-symbol-maintainers script consumes the input from symbol-tool and generates email notifications of expired symbols. v2: reworked to fix pylint errors v3: sent with the correct in-reply-to v4: fix typos picked up by the CI v5: fix terminal_size & directory args v6: added list-expired, to list expired experimental symbols v7: fix typo in comments v8: added tool to notify maintainers of expired symbols v9: removed hardcoded emails addressed and script names Ray Kinsella (2): devtools: script to track symbols over releases devtools: script to send notifications of expired symbols devtools/notify-symbol-maintainers.py | 234 +++++++++++++++ devtools/symbol-tool.py | 402 ++++++++++++++++++++++++++ 2 files changed, 636 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py create mode 100755 devtools/symbol-tool.py -- 2.26.2
This script tracks the growth of stable and experimental symbols over releases since v19.11. The script has the ability to count the added symbols between two dpdk releases, and to list experimental symbols present in two dpdk releases (expired symbols). example usages: Count symbols added since v19.11 $ devtools/symbol-tool.py count-symbols Count symbols added since v20.11 $ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05 List experimental symbols present in v20.11 and v21.05 $ devtools/symbol-tool.py list-expired --releases v20.11,v21.05 List experimental symbols in libraries only, present since v19.11 $ devtools/symbol-tool.py list-expired --directory lib Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/symbol-tool.py | 402 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 402 insertions(+) create mode 100755 devtools/symbol-tool.py diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py new file mode 100755 index 0000000000..4a357579dc --- /dev/null +++ b/devtools/symbol-tool.py @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +'''Tool to count or list symbols in each DPDK release''' +from pathlib import Path +import sys +import os +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import re +import datetime +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + sys.exit() + +DESCRIPTION = ''' +This script tracks the growth of stable and experimental symbols +over releases since v19.11. The script has the ability to +count the added symbols between two dpdk releases, and to +list experimental symbols present in two dpdk releases +(expired symbols). + +example usages: + +Count symbols added since v19.11 +$ {s} count-symbols + +Count symbols added since v20.11 +$ {s} count-symbols --releases v20.11,v21.05 + +List experimental symbols present in v20.11 and v21.05 +$ {s} list-expired --releases v20.11,v21.05 + +List experimental symbols in libraries only, present since v19.11 +$ {s} list-expired --directory lib +''' + +MAP_GRAMMAR = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" + +def get_abi_versions(): + '''Returns a string of possible dpdk abi versions''' + + year = datetime.date.today().year - 2000 + tags = " |".join(['\'{}\''.format(i) \ + for i in reversed(range(21, year + 1)) ]) + tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return tags + +def get_dpdk_releases(): + '''Returns a list of dpdk release tags names since v19.11''' + + year = datetime.date.today().year - 2000 + year_range = "|".join("{}".format(i) for i in range(19,year + 1)) + pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + try: + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + print("Failed to interogate git for release tags") + sys.exit() + + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [ tag.replace('\"','') \ + for tag in reversed(tags) \ + if pattern.match(tag) ][:-3] + + return tags + +def fix_directory_name(path): + '''Prepend librte to the source directory name''' + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + +def directory_renamed(path, rel): + '''Fix removal of the librte_ from the directory names''' + + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + return result + +def mapfile_renamed(path, rel): + '''Fix renaming of the map file''' + newfile = None + + result = subprocess.run(['git', 'ls-tree', \ + rel, str(path.parent) + '/'], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if newfile is not None: + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + else: + result = None + + return result + +def mapfile_and_directory_renamed(path, rel): + '''Fix renaming of the map file & the source directory''' + mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name)) + + return mapfile_renamed(mapfilepath, rel) + +FIX_STRATEGIES = [directory_renamed, \ + mapfile_renamed, \ + mapfile_and_directory_renamed] + +def get_symbols(map_parser, release, mapfile_path): + '''Count the symbols for a given release and mapfile''' + abi_sections = {} + + tagfile = '{}:{}'.format(release,mapfile_path) + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + for fix_strategy in FIX_STRATEGIES: + if result is not None: + break + result = fix_strategy(mapfile_path, release) + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = map_parser(mapfile).abi() + + return abi_sections + +def get_terminal_rows(): + '''Find the number of rows in the terminal''' + + try: + return os.get_terminal_size().lines + except IOError: + return 0 + +class SymbolCountOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.OUTPUT_FORMATS[format_output](self,dpdk_releases) + self.column_titles = ['mapfile'] + dpdk_releases + + self.terminal_rows = get_terminal_rows() + self.row = 0 + + def set_terminal_output(self,dpdk_rel): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}' + \ + ''.join(['{:<6}{:<6}'] * (len(dpdk_rel))) + self.column_fmt = '{:50}' + \ + ''.join(['{:<12}'] * (len(dpdk_rel))) + + def set_csv_output(self,dpdk_rel): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},' + \ + ','.join(['{},{}'] * (len(dpdk_rel))) + self.column_fmt = '{},' + \ + ','.join(['{},'] * (len(dpdk_rel))) + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + self.row += 1 + + def print_row(self, mapfile, symbols): + '''Print row of symbol values''' + print(self.output_fmt.format(*([mapfile] + symbols))) + self.row += 1 + + if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)): + self.print_columns() + + OUTPUT_FORMATS = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } + +class ListExpiredOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.terminal = True + self.OUTPUT_FORMATS[format_output](self,dpdk_releases) + self.column_titles = ['mapfile'] + \ + ['expired (' + ','.join(dpdk_releases) + ')'] + + def set_terminal_output(self, _): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}{:<50}' + self.column_fmt = '{:50}{:50}' + + def set_csv_output(self, _): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},{}' + self.column_fmt = '{},{}' + self.terminal = False + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + + def print_row(self, mapfile, symbols): + '''Print row of symbol values''' + + for symbol in symbols: + print(self.output_fmt.format(mapfile,symbol)) + if self.terminal : + mapfile = '' + + OUTPUT_FORMATS = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } + +class CountSymbolsAction: + ''' Logic to count symbols added since a give release ''' + IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL'] + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.symbols_count = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbol_count = experimental_count = 0 + + symbols = get_symbols(self.parser, release, self.path) + + # which versions are present, and we care about + abi_vers = [abi_ver \ + for abi_ver in symbols \ + if abi_ver not in self.IGNORE_SECTIONS] + + for abi_ver in abi_vers: + symbol_count += len(symbols[abi_ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in symbols.keys(): + experimental_count = len(symbols['EXPERIMENTAL']) + + self.symbols_count += [symbol_count, experimental_count] + + def __del__(self): + self.format_output.print_row(self.path.parent, self.symbols_count) + +class ListExpiredAction: + ''' Logic to list expired symbols between two releases ''' + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.experimental_symbols = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbols = get_symbols(self.parser, release, self.path) + if 'EXPERIMENTAL' in symbols.keys(): + self.experimental_symbols.append(symbols['EXPERIMENTAL']) + + def __del__(self): + if len(self.experimental_symbols) != 2: + return + + tmp = self.experimental_symbols + # find symbols present in both dpdk releases + intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]] + + # check for empty set + if intersect_syms == []: + return + + self.format_output.print_row(self.path.parent, intersect_syms) + +SRC_DIRECTORIES = 'drivers,lib' + +ACTIONS = {None: CountSymbolsAction, \ + 'count-symbols': CountSymbolsAction, \ + 'list-expired': ListExpiredAction} + +ACTION_OUTPUT = {None: SymbolCountOutput, \ + 'count-symbols': SymbolCountOutput, \ + 'list-expired': ListExpiredOutput} + +def main(): + '''Main entry point''' + + dpdk_releases = get_dpdk_releases() + + parser = argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), \ + formatter_class=RawTextHelpFormatter + ) + parser.add_argument('mode', choices=['count-symbols','list-expired']) + parser.add_argument('--format-output', choices=['terminal','csv'], \ + default='terminal') + parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','), + default=SRC_DIRECTORIES) + parser.add_argument('--releases', \ + help='2 x comma separated release tags e.g. \'' \ + + ','.join([dpdk_releases[0],dpdk_releases[-1]]) \ + + '\'') + args = parser.parse_args() + + if args.releases is not None: + dpdk_releases = args.releases.split(',') + + if args.mode == 'list-expired': + if len(dpdk_releases) < 2: + sys.exit('Please specify two releases to compare ' \ + 'in \'list-expired\' mode.') + dpdk_releases = [dpdk_releases[0], dpdk_releases[len(dpdk_releases) - 1]] + + action = ACTIONS[args.mode] + format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases) + + map_grammar = MAP_GRAMMAR.format(get_abi_versions()) + map_parser = makeGrammar(map_grammar, {}) + + format_output.print_columns() + + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + release_action = action(path, map_parser, format_output) + + for release in dpdk_releases: + release_action.add_mapfile(release) + + # all the magic happens in the destructor + del release_action + +if __name__ == '__main__': + main() -- 2.26.2
Use this script with the output of the DPDK symbol tool, to notify maintainers of expired symbols by email. You need to define the environment variable DPDK_GETMAINTAINER_PATH for this tool to work. Use terminal output to review the emails before sending. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ devtools/notify_expired_symbols.py --format-output terminal Then use email output to send the emails to the maintainers. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ devtools/notify_expired_symbols.py --format-output email \ --smtp-server <server> --sender <someone@somewhere.com> \ --password <password> Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/notify-symbol-maintainers.py | 234 ++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py new file mode 100755 index 0000000000..a6c27b067c --- /dev/null +++ b/devtools/notify-symbol-maintainers.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +'''Tool to notify maintainers of expired symbols''' +import smtplib +import ssl +import sys +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import time +from email.message import EmailMessage + +DESCRIPTION = ''' +Use this script with the output of the DPDK symbol tool, to notify maintainers +of expired symbols by email. You need to define the environment variable +DPDK_GETMAINTAINER_PATH, for this tool to work. + +Use terminal output to review the emails before sending. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +{s} --format-output terminal + +Then use email output to send the emails to the maintainers. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +{s} --format-output email \\ +--smtp-server <server> --sender <someone@somewhere.com> --password <password> +''' + +EMAIL_TEMPLATE = '''Hi there, + +Please note the symbols listed below have expired. In line with the DPDK ABI +policy, they should be scheduled for removal, in the next DPDK release. + +For more information, please see the DPDK ABI Policy, section 3.5.3. +https://doc.dpdk.org/guides/contributing/abi_policy.html + +Thanks, + +The DPDK Symbol Bot + +''' + +ABI_POLICY = 'doc/guides/contributing/abi_policy.rst' +get_maintainer = ['devtools/get-maintainer.sh', \ + '--email', '-f'] + +def _get_maintainers(libpath): + '''Get the maintainers for given library''' + try: + cmd = get_maintainer + [libpath] + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + return None + + if result is None: + return None + + email = result.stdout.decode('utf-8') + if email == '': + return None + + email = list(filter(None,email.split('\n'))) + return email + +default_maintainers = _get_maintainers(ABI_POLICY) + +def get_maintainers(libpath): + '''Get the maintainers for given library''' + maintainers=_get_maintainers(libpath) + + if maintainers is None: + maintainers = default_maintainers + + return maintainers + +def get_message(library, symbols): + '''Build email message from symbols, config and maintainers''' + message = {} + maintainers = get_maintainers(library) + + message['To'] = maintainers + if maintainers != default_maintainers: + message['CC'] = default_maintainers + + message['Subject'] = 'Expired symbols in {}\n'.format(library) + + body = EMAIL_TEMPLATE + for sym in symbols: + body += ('{}\n'.format(sym)) + + message['Body'] = body + + return message + +class OutputEmail(): + '''Format the output for email''' + def __init__(self, config): + self.config = config + + self.terminal = OutputTerminal(config) + context = ssl.create_default_context() + + # Try to log in to server and send email + try: + self.server = smtplib.SMTP(config['smtp_server'], 587) + self.server.starttls(context=context) # Secure the connection + self.server.login(config['sender'], config['password']) + except Exception as exception: + print(exception) + raise exception + + def message(self,message): + '''send email''' + self.terminal.message(message) + + msg = EmailMessage() + msg.set_content(message.pop('Body')) + + for key in message.keys(): + msg[key] = message[key] + + msg['From'] = self.config['sender'] + msg['Reply-To'] = 'no-reply@dpdk.org' + + self.server.send_message(msg) + + time.sleep(1) + + def __del__(self): + self.server.quit() + +class OutputTerminal(): # pylint: disable=too-few-public-methods + '''Format the output for the terminal''' + def __init__(self, config): + self.config = config + + def message(self,message): + '''Print email to terminal''' + terminal = 'To:' + ', '.join(message['To']) + '\n' + if 'sender' in self.config.keys(): + terminal += 'From:' + self.config['sender'] + '\n' + + terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n' + if 'CC' in message.keys(): + terminal += 'CC:' + ', '.join(message['CC']) + '\n' + + terminal += 'Subject:' + message['Subject'] + '\n' + terminal += 'Body:' + message['Body'] + '\n' + + print(terminal) + print('-' * 80) + +def parse_config(args): + '''put the command line args in the right places''' + config = {} + error_msg = None + + outputs = { + None : OutputTerminal, + 'terminal' : OutputTerminal, + 'email' : OutputEmail + } + + if args.format_output == 'email': + if args.smtp_server is None: + error_msg = 'SMTP server' + else: + config['smtp_server'] = args.smtp_server + + if args.sender is None: + error_msg = 'sender' + else: + config['sender'] = args.sender + + if args.password is None: + error_msg = 'password' + else: + config['password'] = args.password + + if error_msg is not None: + print('Please specify a {} for email output'.format(error_msg)) + return None + + config['output'] = outputs[args.format_output] + return config + +def main(): + '''Main entry point''' + parser = argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), \ + formatter_class=RawTextHelpFormatter) + parser.add_argument('--format-output', choices=['terminal','email'], \ + default='terminal') + parser.add_argument('--smtp-server') + parser.add_argument('--password') + parser.add_argument('--sender') + + args = parser.parse_args() + config = parse_config(args) + if config is None: + return + + symbols = [] + lastlib = library = '' + + output = config['output'](config) + + for line in sys.stdin: + line = line.rstrip('\n') + library, symbol = [line[:line.find(',')], \ + line[line.find(',') + 1: len(line)]] + if library == 'mapfile': + continue + + if library != lastlib: + message = get_message(lastlib, symbols) + output.message(message) + symbols = [] + + lastlib = library + symbols = symbols + [symbol] + + #print the last library + message = get_message(lastlib, symbols) + output.message(message) + +if __name__ == '__main__': + main() -- 2.26.2
Scripts to count and track the lifecycle of DPDK symbols. The symbol-tool script reports on the growth of symbols over releases and list expired symbols. The notify-symbol-maintainers script consumes the input from symbol-tool and generates email notifications of expired symbols. v2: reworked to fix pylint errors v3: sent with the correct in-reply-to v4: fix typos picked up by the CI v5: fix terminal_size & directory args v6: added list-expired, to list expired experimental symbols v7: fix typo in comments v8: added tool to notify maintainers of expired symbols v9: removed hardcoded emails addressed and script names v10: added ability to identify and notify the original contributors Ray Kinsella (3): devtools: script to track symbols over releases devtools: script to send notifications of expired symbols maintainers: add new abi scripts MAINTAINERS | 2 + devtools/notify-symbol-maintainers.py | 256 ++++++++++++++ devtools/symbol-tool.py | 482 ++++++++++++++++++++++++++ 3 files changed, 740 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py create mode 100755 devtools/symbol-tool.py -- 2.26.2
This script tracks the growth of stable and experimental symbols over releases since v19.11. The script has the ability to count the added symbols between two dpdk releases, and to list experimental symbols present in two dpdk releases (expired symbols). example usages: Count symbols added since v19.11 $ devtools/symbol-tool.py count-symbols Count symbols added since v20.11 $ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05 List experimental symbols present in v20.11 and v21.05 $ devtools/symbol-tool.py list-expired --releases v20.11,v21.05 List experimental symbols in libraries only, present since v19.11 $ devtools/symbol-tool.py list-expired --directory lib Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/symbol-tool.py | 482 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 482 insertions(+) create mode 100755 devtools/symbol-tool.py diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py new file mode 100755 index 0000000000..3d093a0802 --- /dev/null +++ b/devtools/symbol-tool.py @@ -0,0 +1,482 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +# pylint: disable=invalid-name +'''Tool to count or list symbols in each DPDK release''' +from pathlib import Path +import sys +import os +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import re +import datetime +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + sys.exit() + +DESCRIPTION = ''' +This script tracks the growth of stable and experimental symbols +over releases since v19.11. The script has the ability to +count the added symbols between two dpdk releases, and to +list experimental symbols present in two dpdk releases +(expired symbols), including the name & email of the original contributor. + +example usages: + +Count symbols added since v19.11 +$ {s} count-symbols + +Count symbols added since v20.11 +$ {s} count-symbols --releases v20.11,v21.05 + +List experimental symbols present in v20.11 and v21.05 +$ {s} list-expired --releases v20.11,v21.05 + +List experimental symbols in libraries only, present since v19.11 +$ {s} list-expired --directory lib +''' + +MAP_GRAMMAR = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" + +def get_abi_versions(): + '''Returns a string of possible dpdk abi versions''' + + year = datetime.date.today().year - 2000 + tags = " |".join(['\'{}\''.format(i) \ + for i in reversed(range(21, year + 1)) ]) + tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return tags + +def get_dpdk_releases(): + '''Returns a list of dpdk release tags names since v19.11''' + + year = datetime.date.today().year - 2000 + year_range = "|".join("{}".format(i) for i in range(19,year + 1)) + pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + try: + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + print("Failed to interogate git for release tags") + sys.exit() + + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [ tag.replace('\"','') \ + for tag in reversed(tags) \ + if pattern.match(tag) ][:-3] + + return tags + +def fix_directory_name(path): + '''Prepend librte to the source directory name''' + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + +def directory_renamed(path, rel): + '''Fix removal of the librte_ from the directory names''' + + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + return result + +def mapfile_renamed(path, rel): + '''Fix renaming of the map file''' + newfile = None + + result = subprocess.run(['git', 'ls-tree', \ + rel, str(path.parent) + '/'], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if newfile is not None: + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + else: + result = None + + return result + +def mapfile_and_directory_renamed(path, rel): + '''Fix renaming of the map file & the source directory''' + mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name)) + + return mapfile_renamed(mapfilepath, rel) + +FIX_STRATEGIES = [directory_renamed, \ + mapfile_renamed, \ + mapfile_and_directory_renamed] + +def get_symbols(map_parser, release, mapfile_path): + '''Count the symbols for a given release and mapfile''' + abi_sections = {} + + tagfile = '{}:{}'.format(release,mapfile_path) + try: + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + for fix_strategy in FIX_STRATEGIES: + if result is not None: + break + result = fix_strategy(mapfile_path, release) + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = map_parser(mapfile).abi() + + return abi_sections + +def get_terminal_rows(): + '''Find the number of rows in the terminal''' + + try: + return os.get_terminal_size().lines + except IOError: + return 0 + +class SymbolOwner(): + '''Find the symbols original contributors name and email''' + symbol_regex = {} + blame_regex = {'name' : r'author\s(.*)', \ + 'email' : r'author-mail\s<(.*)>'} + + def __init__(self, libpath, symbol): + self.libpath = libpath + self.symbol = symbol + + #find variable definitions in C files, and functions in headers. + self.symbol_regex = \ + {'*.c' : r'^(?!extern).*' + self.symbol + '[^()]*;', \ + '*.h' : r'__rte_experimental(?:.*\n){0,2}.*' + self.symbol} + + def find_symbol_location(self): + '''Find where the symbol is definited in the source''' + for key in self.symbol_regex: + for path in Path(self.libpath).rglob(key): + file_text = open(path).read() + + #find where the symbol is defined, either preceeded by + #rte_experimental tag (functions) or followed by a ; (variables) + + exp = self.symbol_regex[key] + pattern = re.compile(exp, re.MULTILINE) + search = pattern.search(file_text) + + if search is not None: + symbol_pos = search.span()[1] + symbol_line = file_text.count('\n', 0, symbol_pos) + 1 + + return [str(path),symbol_line] + return None + + def find_symbol_owner(self): + '''Find the symbols original contributors name and email''' + owners = {} + location = self.find_symbol_location() + + if location is None: + return None + + line = '-L {},{}'.format(location[1],location[1]) + #git blame -p(orcelain) -L(ine) path + args = ['-p', line, location[0]] + + try: + result = subprocess.run(['git', 'blame'] + args, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + return None + + blame = result.stdout.decode('utf-8') + for key in self.blame_regex: + pattern = re.compile(self.blame_regex[key], re.MULTILINE) + match = pattern.search(blame) + + owners[key] = match.groups()[0] + + return owners + +class SymbolCountOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.OUTPUT_FORMATS[format_output](self,dpdk_releases) + self.column_titles = ['mapfile'] + dpdk_releases + + self.terminal_rows = get_terminal_rows() + self.row = 0 + + def set_terminal_output(self,dpdk_rel): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}' + \ + ''.join(['{:<6}{:<6}'] * (len(dpdk_rel))) + self.column_fmt = '{:50}' + \ + ''.join(['{:<12}'] * (len(dpdk_rel))) + + def set_csv_output(self,dpdk_rel): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},' + \ + ','.join(['{},{}'] * (len(dpdk_rel))) + self.column_fmt = '{},' + \ + ','.join(['{},'] * (len(dpdk_rel))) + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + self.row += 1 + + def print_row(self, mapfile, symbols): + '''Print row of symbol values''' + mapfile = str(mapfile) + print(self.output_fmt.format(*([mapfile] + symbols))) + self.row += 1 + + if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)): + self.print_columns() + + OUTPUT_FORMATS = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } + +class ListExpiredOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.terminal = True + self.OUTPUT_FORMATS[format_output](self,dpdk_releases) + self.column_titles = ['mapfile'] + \ + ['expired (' + ','.join(dpdk_releases) + ')'] + \ + ['contributor name', 'contributor email'] + + def set_terminal_output(self, _): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}{:<50}{:<25}{:<25}' + self.column_fmt = '{:50}{:50}{:25}{:25}' + + def set_csv_output(self, _): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},{},{},{}' + self.column_fmt = '{},{},{},{}' + self.terminal = False + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + + def print_row(self, mapfile, symbols, owner): + '''Print row of symbol values''' + + for symbol in symbols: + mapfile = str(mapfile) + name = owner[symbol]['name'] \ + if owner[symbol] is not None else '' + email = owner[symbol]['email'] \ + if owner[symbol] is not None else '' + + print(self.output_fmt.format(mapfile, symbol, name, email)) + if self.terminal: + mapfile = '' + + OUTPUT_FORMATS = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } + +class CountSymbolsAction: + ''' Logic to count symbols added since a give release ''' + IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL'] + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.symbols_count = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbol_count = experimental_count = 0 + + symbols = get_symbols(self.parser, release, self.path) + + # which versions are present, and we care about + abi_vers = [abi_ver \ + for abi_ver in symbols \ + if abi_ver not in self.IGNORE_SECTIONS] + + for abi_ver in abi_vers: + symbol_count += len(symbols[abi_ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in symbols.keys(): + experimental_count = len(symbols['EXPERIMENTAL']) + + self.symbols_count += [symbol_count, experimental_count] + + def __del__(self): + self.format_output.print_row(self.path.parent, self.symbols_count) + +class ListExpiredAction: + ''' Logic to list expired symbols between two releases ''' + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.experimental_symbols = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbols = get_symbols(self.parser, release, self.path) + + if 'EXPERIMENTAL' in symbols.keys(): + experimental = [exp.strip() for exp in symbols['EXPERIMENTAL']] + + self.experimental_symbols.append(experimental) + + def __del__(self): + if len(self.experimental_symbols) != 2: + return + + tmp = self.experimental_symbols + # find symbols present in both dpdk releases + intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]] + + # check for empty set + if intersect_syms == []: + return + + sym_owner = {} + for sym in intersect_syms: + sym_owner[sym] = SymbolOwner(self.path.parent, sym).find_symbol_owner() + + self.format_output.print_row(self.path.parent, intersect_syms, sym_owner) + +SRC_DIRECTORIES = 'drivers,lib' + +ACTIONS = {None: CountSymbolsAction, \ + 'count-symbols': CountSymbolsAction, \ + 'list-expired': ListExpiredAction} + +ACTION_OUTPUT = {None: SymbolCountOutput, \ + 'count-symbols': SymbolCountOutput, \ + 'list-expired': ListExpiredOutput} + +def main(): + '''Main entry point''' + + dpdk_releases = get_dpdk_releases() + + parser = argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), \ + formatter_class=RawTextHelpFormatter + ) + parser.add_argument('mode', choices=['count-symbols','list-expired']) + parser.add_argument('--format-output', choices=['terminal','csv'], \ + default='terminal') + parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','), + default=SRC_DIRECTORIES) + parser.add_argument('--releases', \ + help='2 x comma separated release tags e.g. \'' \ + + ','.join([dpdk_releases[0],dpdk_releases[-1]]) \ + + '\'') + args = parser.parse_args() + + if args.releases is not None: + dpdk_releases = args.releases.split(',') + + if args.mode == 'list-expired': + if len(dpdk_releases) < 2: + sys.exit('Please specify two releases to compare ' \ + 'in \'list-expired\' mode.') + dpdk_releases = [dpdk_releases[0], dpdk_releases[len(dpdk_releases) - 1]] + + action = ACTIONS[args.mode] + format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases) + + map_grammar = MAP_GRAMMAR.format(get_abi_versions()) + map_parser = makeGrammar(map_grammar, {}) + + format_output.print_columns() + + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + release_action = action(path, map_parser, format_output) + + for release in dpdk_releases: + release_action.add_mapfile(release) + + # all the magic happens in the destructor + del release_action + +if __name__ == '__main__': + main() -- 2.26.2
Use this script with the output of the DPDK symbol tool, to notify maintainers of expired symbols by email. You need to define the environment variable DPDK_GETMAINTAINER_PATH for this tool to work. Use terminal output to review the emails before sending. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ devtools/notify_expired_symbols.py --format-output terminal Then use email output to send the emails to the maintainers. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ devtools/notify_expired_symbols.py --format-output email \ --smtp-server <server> --sender <someone@somewhere.com> \ --password <password> --cc <someone@somewhere.com> Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/notify-symbol-maintainers.py | 256 ++++++++++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py new file mode 100755 index 0000000000..ee554687ff --- /dev/null +++ b/devtools/notify-symbol-maintainers.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +# pylint: disable=invalid-name +'''Tool to notify maintainers of expired symbols''' +import smtplib +import ssl +import sys +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import time +from email.message import EmailMessage + +DESCRIPTION = ''' +Use this script with the output of the DPDK symbol tool, to notify maintainers +and contributors of expired symbols by email. You need to define the environment +variable DPDK_GETMAINTAINER_PATH for this tool to work. + +Use terminal output to review the emails before sending. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +{s} --format-output terminal + +Then use email output to send the emails to the maintainers. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +{s} --format-output email \\ +--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\ +--cc <someone@somewhere.com> +''' + +EMAIL_TEMPLATE = '''Hi there, + +Please note the symbols listed below have expired. In line with the DPDK ABI +policy, they should be scheduled for removal, in the next DPDK release. + +For more information, please see the DPDK ABI Policy, section 3.5.3. +https://doc.dpdk.org/guides/contributing/abi_policy.html + +Thanks, + +The DPDK Symbol Bot + +''' + +ABI_POLICY = 'doc/guides/contributing/abi_policy.rst' +MAINTAINERS = 'MAINTAINERS' +get_maintainer = ['devtools/get-maintainer.sh', \ + '--email', '-f'] + +def _get_maintainers(libpath): + '''Get the maintainers for given library''' + try: + cmd = get_maintainer + [libpath] + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + return None + + if result is None: + return None + + email = result.stdout.decode('utf-8') + if email == '': + return None + + email = list(filter(None,email.split('\n'))) + return email + +default_maintainers = _get_maintainers(ABI_POLICY) + \ + _get_maintainers(MAINTAINERS) + +def get_maintainers(libpath): + '''Get the maintainers for given library''' + maintainers=_get_maintainers(libpath) + + if maintainers is None: + maintainers = default_maintainers + + return maintainers + +def get_message(library, symbols, config): + '''Build email message from symbols, config and maintainers''' + contributors = {} + message = {} + maintainers = get_maintainers(library) + + if maintainers != default_maintainers: + message['CC'] = default_maintainers.copy() + + if 'CC' in config: + message.setdefault('CC',[]).append(config['CC']) + + message['Subject'] = 'Expired symbols in {}\n'.format(library) + + body = EMAIL_TEMPLATE + body += '{:<50}{:<25}{:<25}\n'.format('Symbol','Contributor','Email') + for sym in symbols: + body += ('{:<50}{:<25}{:<25}\n'.format(sym,\ + symbols[sym]['name'], + symbols[sym]['email'], + )) + email = symbols[sym]['email'] + contributors[email] = '' + + contributors = list(contributors.keys()) + + message['To'] = maintainers + contributors + message['Body'] = body + + return message + +class OutputEmail(): + '''Format the output for email''' + def __init__(self, config): + self.config = config + + self.terminal = OutputTerminal(config) + context = ssl.create_default_context() + + # Try to log in to server and send email + try: + self.server = smtplib.SMTP(config['smtp_server'], 587) + self.server.starttls(context=context) # Secure the connection + self.server.login(config['sender'], config['password']) + except Exception as exception: + print(exception) + raise exception + + def message(self,message): + '''send email''' + self.terminal.message(message) + + msg = EmailMessage() + msg.set_content(message.pop('Body')) + + for key in message.keys(): + msg[key] = message[key] + + msg['From'] = self.config['sender'] + msg['Reply-To'] = 'no-reply@dpdk.org' + + self.server.send_message(msg) + + time.sleep(1) + + def __del__(self): + self.server.quit() + +class OutputTerminal(): # pylint: disable=too-few-public-methods + '''Format the output for the terminal''' + def __init__(self, config): + self.config = config + + def message(self,message): + '''Print email to terminal''' + + terminal = 'To:' + ', '.join(message['To']) + '\n' + if 'sender' in self.config.keys(): + terminal += 'From:' + self.config['sender'] + '\n' + + terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n' + + if 'CC' in message: + terminal += 'CC:' + ', '.join(message['CC']) + '\n' + + terminal += 'Subject:' + message['Subject'] + '\n' + terminal += 'Body:' + message['Body'] + '\n' + + print(terminal) + print('-' * 80) + +def parse_config(args): + '''put the command line args in the right places''' + config = {} + error_msg = None + + outputs = { + None : OutputTerminal, + 'terminal' : OutputTerminal, + 'email' : OutputEmail + } + + if args.format_output == 'email': + if args.smtp_server is None: + error_msg = 'SMTP server' + else: + config['smtp_server'] = args.smtp_server + + if args.sender is None: + error_msg = 'sender' + else: + config['sender'] = args.sender + + if args.password is None: + error_msg = 'password' + else: + config['password'] = args.password + + if args.cc is not None: + config['CC'] = args.cc + + if error_msg is not None: + print('Please specify a {} for email output'.format(error_msg)) + return None + + config['output'] = outputs[args.format_output] + return config + +def main(): + '''Main entry point''' + parser = argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), \ + formatter_class=RawTextHelpFormatter) + parser.add_argument('--format-output', choices=['terminal','email'], \ + default='terminal') + parser.add_argument('--smtp-server') + parser.add_argument('--password') + parser.add_argument('--sender') + parser.add_argument('--cc') + + args = parser.parse_args() + config = parse_config(args) + if config is None: + return + + symbols = {} + lastlib = library = '' + + output = config['output'](config) + + for line in sys.stdin: + line = line.rstrip('\n') + + if line.find('mapfile') >= 0: + continue + library, symbol, name, email = line.split(',') + + if library != lastlib: + message = get_message(lastlib, symbols, config) + output.message(message) + symbols = {} + + lastlib = library + symbols[symbol] = {'name' : name, 'email' : email} + + #print the last library + message = get_message(lastlib, symbols, config) + output.message(message) + +if __name__ == '__main__': + main() -- 2.26.2
Add new abi management scripts to the MAINTAINERS file. Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 266f5ac1da..ff8245271f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -129,6 +129,8 @@ F: devtools/gen-abi.sh F: devtools/libabigail.abignore F: devtools/update-abi.sh F: devtools/update_version_map_abi.py +F: devtools/notify-symbol-maintainers.py +F: devtools/symbol-tool.py F: buildtools/check-symbols.sh F: buildtools/map-list-symbol.sh F: drivers/*/*/*.map -- 2.26.2
Ray Kinsella <mdr@ashroe.eu> writes:
> Scripts to count and track the lifecycle of DPDK symbols.
>
> The symbol-tool script reports on the growth of symbols over releases
> and list expired symbols. The notify-symbol-maintainers script
> consumes the input from symbol-tool and generates email notifications
> of expired symbols.
>
> v2: reworked to fix pylint errors
> v3: sent with the correct in-reply-to
> v4: fix typos picked up by the CI
> v5: fix terminal_size & directory args
> v6: added list-expired, to list expired experimental symbols
> v7: fix typo in comments
> v8: added tool to notify maintainers of expired symbols
> v9: removed hardcoded emails addressed and script names
> v10: added ability to identify and notify the original contributors
>
> Ray Kinsella (3):
> devtools: script to track symbols over releases
> devtools: script to send notifications of expired symbols
> maintainers: add new abi scripts
>
> MAINTAINERS | 2 +
> devtools/notify-symbol-maintainers.py | 256 ++++++++++++++
> devtools/symbol-tool.py | 482 ++++++++++++++++++++++++++
> 3 files changed, 740 insertions(+)
> create mode 100755 devtools/notify-symbol-maintainers.py
> create mode 100755 devtools/symbol-tool.py
I get a whole mess of flake8 issues from this series (mostly 'backslash
is redundant' and whitespace issues). I'm using flake8 because it
pretty well enforces PEP8 style guide. I would like to see it
addressed, but also I see that many of the python files in the DPDK tree
don't actually pass. Example::
$ flake8 ./usertools/dpdk-devbind.py | wc -l
34
$ flake8 ./usertools/dpdk-devbind.py | sed 's@./usertools/dpdk-devbind.py[:0-9]* @@' | sort -u
E128 continuation line under-indented for visual indent
E302 expected 2 blank lines, found 1
E305 expected 2 blank lines after class or function definition, found 1
E501 line too long (105 > 79 characters)
E501 line too long (80 > 79 characters)
E501 line too long (82 > 79 characters)
E501 line too long (83 > 79 characters)
E501 line too long (84 > 79 characters)
E501 line too long (85 > 79 characters)
E501 line too long (86 > 79 characters)
E501 line too long (91 > 79 characters)
E502 the backslash is redundant between brackets
E722 do not use bare 'except'
Looks like we repeat the same kinds of errors everywhere (this is on
multiple tools). Some of our in-tree python is better than others (like
app/test/autotest.py which only has 1 flake).
Maybe we can address this. Other comments inline on the patches.
Ray Kinsella <mdr@ashroe.eu> writes: > Use this script with the output of the DPDK symbol tool, to notify > maintainers of expired symbols by email. You need to define the environment > variable DPDK_GETMAINTAINER_PATH for this tool to work. > > Use terminal output to review the emails before sending. > e.g. > $ devtools/symbol-tool.py list-expired --format-output csv \ > | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ > devtools/notify_expired_symbols.py --format-output terminal > > Then use email output to send the emails to the maintainers. > e.g. > $ devtools/symbol-tool.py list-expired --format-output csv \ > | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ > devtools/notify_expired_symbols.py --format-output email \ > --smtp-server <server> --sender <someone@somewhere.com> \ > --password <password> --cc <someone@somewhere.com> > > Signed-off-by: Ray Kinsella <mdr@ashroe.eu> > --- > devtools/notify-symbol-maintainers.py | 256 ++++++++++++++++++++++++++ > 1 file changed, 256 insertions(+) > create mode 100755 devtools/notify-symbol-maintainers.py > > diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py > new file mode 100755 > index 0000000000..ee554687ff > --- /dev/null > +++ b/devtools/notify-symbol-maintainers.py > @@ -0,0 +1,256 @@ > +#!/usr/bin/env python3 > +# SPDX-License-Identifier: BSD-3-Clause > +# Copyright(c) 2021 Intel Corporation > +# pylint: disable=invalid-name > +'''Tool to notify maintainers of expired symbols''' > +import smtplib > +import ssl > +import sys > +import subprocess > +import argparse > +from argparse import RawTextHelpFormatter > +import time > +from email.message import EmailMessage > + > +DESCRIPTION = ''' > +Use this script with the output of the DPDK symbol tool, to notify maintainers > +and contributors of expired symbols by email. You need to define the environment > +variable DPDK_GETMAINTAINER_PATH for this tool to work. > + > +Use terminal output to review the emails before sending. > +e.g. > +$ devtools/symbol-tool.py list-expired --format-output csv \\ > +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ > +{s} --format-output terminal > + > +Then use email output to send the emails to the maintainers. > +e.g. > +$ devtools/symbol-tool.py list-expired --format-output csv \\ > +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ > +{s} --format-output email \\ > +--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\ > +--cc <someone@somewhere.com> > +''' > + > +EMAIL_TEMPLATE = '''Hi there, > + > +Please note the symbols listed below have expired. In line with the DPDK ABI > +policy, they should be scheduled for removal, in the next DPDK release. > + > +For more information, please see the DPDK ABI Policy, section 3.5.3. > +https://doc.dpdk.org/guides/contributing/abi_policy.html > + > +Thanks, > + > +The DPDK Symbol Bot > + > +''' > + > +ABI_POLICY = 'doc/guides/contributing/abi_policy.rst' > +MAINTAINERS = 'MAINTAINERS' > +get_maintainer = ['devtools/get-maintainer.sh', \ > + '--email', '-f'] Maybe it's best to make this something that can be overridden. There's a series to change the .sh files to .py files. Perhaps an environment variable or argument? > +def _get_maintainers(libpath): > + '''Get the maintainers for given library''' > + try: > + cmd = get_maintainer + [libpath] > + result = subprocess.run(cmd, \ > + stdout=subprocess.PIPE, \ > + stderr=subprocess.PIPE, > + check=True) > + except subprocess.CalledProcessError: > + return None You might consider handling except FileNotFoundError: .... With a graceful exit and error message. In case the get_maintainers path changes. > + if result is None: > + return None > + > + email = result.stdout.decode('utf-8') > + if email == '': > + return None > + > + email = list(filter(None,email.split('\n'))) > + return email > + > +default_maintainers = _get_maintainers(ABI_POLICY) + \ > + _get_maintainers(MAINTAINERS) > + > +def get_maintainers(libpath): > + '''Get the maintainers for given library''' > + maintainers=_get_maintainers(libpath) > + > + if maintainers is None: > + maintainers = default_maintainers > + > + return maintainers > + > +def get_message(library, symbols, config): > + '''Build email message from symbols, config and maintainers''' > + contributors = {} > + message = {} > + maintainers = get_maintainers(library) > + > + if maintainers != default_maintainers: > + message['CC'] = default_maintainers.copy() > + > + if 'CC' in config: > + message.setdefault('CC',[]).append(config['CC']) > + > + message['Subject'] = 'Expired symbols in {}\n'.format(library) > + > + body = EMAIL_TEMPLATE > + body += '{:<50}{:<25}{:<25}\n'.format('Symbol','Contributor','Email') > + for sym in symbols: > + body += ('{:<50}{:<25}{:<25}\n'.format(sym,\ > + symbols[sym]['name'], > + symbols[sym]['email'], > + )) > + email = symbols[sym]['email'] > + contributors[email] = '' > + > + contributors = list(contributors.keys()) > + > + message['To'] = maintainers + contributors > + message['Body'] = body > + > + return message > + > +class OutputEmail(): > + '''Format the output for email''' > + def __init__(self, config): > + self.config = config > + > + self.terminal = OutputTerminal(config) > + context = ssl.create_default_context() > + > + # Try to log in to server and send email > + try: > + self.server = smtplib.SMTP(config['smtp_server'], 587) > + self.server.starttls(context=context) # Secure the connection > + self.server.login(config['sender'], config['password']) > + except Exception as exception: > + print(exception) > + raise exception > + > + def message(self,message): > + '''send email''' > + self.terminal.message(message) > + > + msg = EmailMessage() > + msg.set_content(message.pop('Body')) > + > + for key in message.keys(): > + msg[key] = message[key] > + > + msg['From'] = self.config['sender'] > + msg['Reply-To'] = 'no-reply@dpdk.org' > + > + self.server.send_message(msg) > + > + time.sleep(1) Why this sleep is needed? > + > + def __del__(self): > + self.server.quit() > + > +class OutputTerminal(): # pylint: disable=too-few-public-methods > + '''Format the output for the terminal''' > + def __init__(self, config): > + self.config = config > + > + def message(self,message): > + '''Print email to terminal''' > + > + terminal = 'To:' + ', '.join(message['To']) + '\n' > + if 'sender' in self.config.keys(): > + terminal += 'From:' + self.config['sender'] + '\n' > + > + terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n' > + > + if 'CC' in message: > + terminal += 'CC:' + ', '.join(message['CC']) + '\n' > + > + terminal += 'Subject:' + message['Subject'] + '\n' > + terminal += 'Body:' + message['Body'] + '\n' > + > + print(terminal) > + print('-' * 80) > + > +def parse_config(args): > + '''put the command line args in the right places''' > + config = {} > + error_msg = None > + > + outputs = { > + None : OutputTerminal, > + 'terminal' : OutputTerminal, > + 'email' : OutputEmail > + } > + > + if args.format_output == 'email': > + if args.smtp_server is None: > + error_msg = 'SMTP server' > + else: > + config['smtp_server'] = args.smtp_server > + > + if args.sender is None: > + error_msg = 'sender' > + else: > + config['sender'] = args.sender > + > + if args.password is None: > + error_msg = 'password' > + else: > + config['password'] = args.password > + > + if args.cc is not None: > + config['CC'] = args.cc > + > + if error_msg is not None: > + print('Please specify a {} for email output'.format(error_msg)) > + return None > + > + config['output'] = outputs[args.format_output] > + return config > + > +def main(): > + '''Main entry point''' > + parser = argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), \ > + formatter_class=RawTextHelpFormatter) > + parser.add_argument('--format-output', choices=['terminal','email'], \ > + default='terminal') > + parser.add_argument('--smtp-server') > + parser.add_argument('--password') > + parser.add_argument('--sender') > + parser.add_argument('--cc') > + > + args = parser.parse_args() > + config = parse_config(args) > + if config is None: > + return > + > + symbols = {} > + lastlib = library = '' > + > + output = config['output'](config) > + > + for line in sys.stdin: > + line = line.rstrip('\n') > + > + if line.find('mapfile') >= 0: > + continue > + library, symbol, name, email = line.split(',') > + > + if library != lastlib: > + message = get_message(lastlib, symbols, config) > + output.message(message) > + symbols = {} > + > + lastlib = library > + symbols[symbol] = {'name' : name, 'email' : email} > + > + #print the last library > + message = get_message(lastlib, symbols, config) > + output.message(message) > + > +if __name__ == '__main__': > + main()
Hello Ray, On Tue, Aug 31, 2021 at 4:51 PM Ray Kinsella <mdr@ashroe.eu> wrote: > > Use this script with the output of the DPDK symbol tool, to notify > maintainers of expired symbols by email. You need to define the environment > variable DPDK_GETMAINTAINER_PATH for this tool to work. > > Use terminal output to review the emails before sending. Two comments: - there are references of a previous name for the script, %s/notify_expired_symbols.py/notify-symbol-maintainers.py/g - and a reminder for the empty report that we received yesterday. I think this can be reproduced with: $ DPDK_GETMAINTAINER_PATH=devtools/get_maintainer.pl devtools/notify-symbol-maintainers.py --format-output terminal <<EOF > mapfile,expired (v21.08,v19.11),contributor name,contributor email > lib/rib,rte_rib6_get_ip,Stephen Hemminger,stephen@networkplumber.org > EOF To:Ray Kinsella <mdr@ashroe.eu>, Thomas Monjalon <thomas@monjalon.net> Reply-To:no-reply@dpdk.org Subject:Expired symbols in Body:Hi there, Please note the symbols listed below have expired. In line with the DPDK ABI policy, they should be scheduled for removal, in the next DPDK release. For more information, please see the DPDK ABI Policy, section 3.5.3. https://doc.dpdk.org/guides/contributing/abi_policy.html Thanks, The DPDK Symbol Bot Symbol Contributor Email -------------------------------------------------------------------------------- ^^^^ Here, empty report. To:Vladimir Medvedkin <vladimir.medvedkin@intel.com>, stephen@networkplumber.org Reply-To:no-reply@dpdk.org CC:Ray Kinsella <mdr@ashroe.eu>, Thomas Monjalon <thomas@monjalon.net> Subject:Expired symbols in lib/rib Body:Hi there, Please note the symbols listed below have expired. In line with the DPDK ABI policy, they should be scheduled for removal, in the next DPDK release. For more information, please see the DPDK ABI Policy, section 3.5.3. https://doc.dpdk.org/guides/contributing/abi_policy.html Thanks, The DPDK Symbol Bot Symbol Contributor Email rte_rib6_get_ip Stephen Hemminger stephen@networkplumber.org -------------------------------------------------------------------------------- -- David Marchand
On Wed, 01 Sep 2021 08:31:27 -0400
Aaron Conole <aconole@redhat.com> wrote:
> $ flake8 ./usertools/dpdk-devbind.py | sed 's@./usertools/dpdk-devbind.py[:0-9]* @@' | sort -u
> E128 continuation line under-indented for visual indent
> E302 expected 2 blank lines, found 1
> E305 expected 2 blank lines after class or function definition, found 1
> E501 line too long (105 > 79 characters)
> E501 line too long (80 > 79 characters)
> E501 line too long (82 > 79 characters)
> E501 line too long (83 > 79 characters)
> E501 line too long (84 > 79 characters)
> E501 line too long (85 > 79 characters)
> E501 line too long (86 > 79 characters)
> E501 line too long (91 > 79 characters)
Current practice on many projects has allowed lines up to 100 characters.
Stephen Hemminger <stephen@networkplumber.org> writes:
> On Wed, 01 Sep 2021 08:31:27 -0400
> Aaron Conole <aconole@redhat.com> wrote:
>
>> $ flake8 ./usertools/dpdk-devbind.py | sed 's@./usertools/dpdk-devbind.py[:0-9]* @@' | sort -u
>> E128 continuation line under-indented for visual indent
>> E302 expected 2 blank lines, found 1
>> E305 expected 2 blank lines after class or function definition, found 1
>> E501 line too long (105 > 79 characters)
>> E501 line too long (80 > 79 characters)
>> E501 line too long (82 > 79 characters)
>> E501 line too long (83 > 79 characters)
>> E501 line too long (84 > 79 characters)
>> E501 line too long (85 > 79 characters)
>> E501 line too long (86 > 79 characters)
>> E501 line too long (91 > 79 characters)
>
> Current practice on many projects has allowed lines up to 100 characters.
It is probably okay to run with '--ignore=E501' (which will squelch the
character limit).
On 01/09/2021 13:46, Aaron Conole wrote: > Ray Kinsella <mdr@ashroe.eu> writes: > >> Use this script with the output of the DPDK symbol tool, to notify >> maintainers of expired symbols by email. You need to define the environment >> variable DPDK_GETMAINTAINER_PATH for this tool to work. >> >> Use terminal output to review the emails before sending. >> e.g. >> $ devtools/symbol-tool.py list-expired --format-output csv \ >> | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ >> devtools/notify_expired_symbols.py --format-output terminal >> >> Then use email output to send the emails to the maintainers. >> e.g. >> $ devtools/symbol-tool.py list-expired --format-output csv \ >> | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ >> devtools/notify_expired_symbols.py --format-output email \ >> --smtp-server <server> --sender <someone@somewhere.com> \ >> --password <password> --cc <someone@somewhere.com> >> >> Signed-off-by: Ray Kinsella <mdr@ashroe.eu> >> --- >> devtools/notify-symbol-maintainers.py | 256 ++++++++++++++++++++++++++ >> 1 file changed, 256 insertions(+) >> create mode 100755 devtools/notify-symbol-maintainers.py >> >> diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py >> new file mode 100755 >> index 0000000000..ee554687ff >> --- /dev/null >> +++ b/devtools/notify-symbol-maintainers.py >> @@ -0,0 +1,256 @@ >> +#!/usr/bin/env python3 >> +# SPDX-License-Identifier: BSD-3-Clause >> +# Copyright(c) 2021 Intel Corporation >> +# pylint: disable=invalid-name >> +'''Tool to notify maintainers of expired symbols''' >> +import smtplib >> +import ssl >> +import sys >> +import subprocess >> +import argparse >> +from argparse import RawTextHelpFormatter >> +import time >> +from email.message import EmailMessage >> + >> +DESCRIPTION = ''' >> +Use this script with the output of the DPDK symbol tool, to notify maintainers >> +and contributors of expired symbols by email. You need to define the environment >> +variable DPDK_GETMAINTAINER_PATH for this tool to work. >> + >> +Use terminal output to review the emails before sending. >> +e.g. >> +$ devtools/symbol-tool.py list-expired --format-output csv \\ >> +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ >> +{s} --format-output terminal >> + >> +Then use email output to send the emails to the maintainers. >> +e.g. >> +$ devtools/symbol-tool.py list-expired --format-output csv \\ >> +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ >> +{s} --format-output email \\ >> +--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\ >> +--cc <someone@somewhere.com> >> +''' >> + >> +EMAIL_TEMPLATE = '''Hi there, >> + >> +Please note the symbols listed below have expired. In line with the DPDK ABI >> +policy, they should be scheduled for removal, in the next DPDK release. >> + >> +For more information, please see the DPDK ABI Policy, section 3.5.3. >> +https://doc.dpdk.org/guides/contributing/abi_policy.html >> + >> +Thanks, >> + >> +The DPDK Symbol Bot >> + >> +''' >> + >> +ABI_POLICY = 'doc/guides/contributing/abi_policy.rst' >> +MAINTAINERS = 'MAINTAINERS' >> +get_maintainer = ['devtools/get-maintainer.sh', \ >> + '--email', '-f'] > > Maybe it's best to make this something that can be overridden. There's > a series to change the .sh files to .py files. Perhaps an environment > variable or argument? ACK > >> +def _get_maintainers(libpath): >> + '''Get the maintainers for given library''' >> + try: >> + cmd = get_maintainer + [libpath] >> + result = subprocess.run(cmd, \ >> + stdout=subprocess.PIPE, \ >> + stderr=subprocess.PIPE, >> + check=True) >> + except subprocess.CalledProcessError: >> + return None > > You might consider handling > > except FileNotFoundError: > .... > > With a graceful exit and error message. In case the get_maintainers > path changes. ACK > >> + if result is None: >> + return None >> + >> + email = result.stdout.decode('utf-8') >> + if email == '': >> + return None >> + >> + email = list(filter(None,email.split('\n'))) >> + return email >> + >> +default_maintainers = _get_maintainers(ABI_POLICY) + \ >> + _get_maintainers(MAINTAINERS) >> + >> +def get_maintainers(libpath): >> + '''Get the maintainers for given library''' >> + maintainers=_get_maintainers(libpath) >> + >> + if maintainers is None: >> + maintainers = default_maintainers >> + >> + return maintainers >> + >> +def get_message(library, symbols, config): >> + '''Build email message from symbols, config and maintainers''' >> + contributors = {} >> + message = {} >> + maintainers = get_maintainers(library) >> + >> + if maintainers != default_maintainers: >> + message['CC'] = default_maintainers.copy() >> + >> + if 'CC' in config: >> + message.setdefault('CC',[]).append(config['CC']) >> + >> + message['Subject'] = 'Expired symbols in {}\n'.format(library) >> + >> + body = EMAIL_TEMPLATE >> + body += '{:<50}{:<25}{:<25}\n'.format('Symbol','Contributor','Email') >> + for sym in symbols: >> + body += ('{:<50}{:<25}{:<25}\n'.format(sym,\ >> + symbols[sym]['name'], >> + symbols[sym]['email'], >> + )) >> + email = symbols[sym]['email'] >> + contributors[email] = '' >> + >> + contributors = list(contributors.keys()) >> + >> + message['To'] = maintainers + contributors >> + message['Body'] = body >> + >> + return message >> + >> +class OutputEmail(): >> + '''Format the output for email''' >> + def __init__(self, config): >> + self.config = config >> + >> + self.terminal = OutputTerminal(config) >> + context = ssl.create_default_context() >> + >> + # Try to log in to server and send email >> + try: >> + self.server = smtplib.SMTP(config['smtp_server'], 587) >> + self.server.starttls(context=context) # Secure the connection >> + self.server.login(config['sender'], config['password']) >> + except Exception as exception: >> + print(exception) >> + raise exception >> + >> + def message(self,message): >> + '''send email''' >> + self.terminal.message(message) >> + >> + msg = EmailMessage() >> + msg.set_content(message.pop('Body')) >> + >> + for key in message.keys(): >> + msg[key] = message[key] >> + >> + msg['From'] = self.config['sender'] >> + msg['Reply-To'] = 'no-reply@dpdk.org' >> + >> + self.server.send_message(msg) >> + >> + time.sleep(1) > > Why this sleep is needed? Don't hammer the mail server :-) > >> + >> + def __del__(self): >> + self.server.quit() >> + >> +class OutputTerminal(): # pylint: disable=too-few-public-methods >> + '''Format the output for the terminal''' >> + def __init__(self, config): >> + self.config = config >> + >> + def message(self,message): >> + '''Print email to terminal''' >> + >> + terminal = 'To:' + ', '.join(message['To']) + '\n' >> + if 'sender' in self.config.keys(): >> + terminal += 'From:' + self.config['sender'] + '\n' >> + >> + terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n' >> + >> + if 'CC' in message: >> + terminal += 'CC:' + ', '.join(message['CC']) + '\n' >> + >> + terminal += 'Subject:' + message['Subject'] + '\n' >> + terminal += 'Body:' + message['Body'] + '\n' >> + >> + print(terminal) >> + print('-' * 80) >> + >> +def parse_config(args): >> + '''put the command line args in the right places''' >> + config = {} >> + error_msg = None >> + >> + outputs = { >> + None : OutputTerminal, >> + 'terminal' : OutputTerminal, >> + 'email' : OutputEmail >> + } >> + >> + if args.format_output == 'email': >> + if args.smtp_server is None: >> + error_msg = 'SMTP server' >> + else: >> + config['smtp_server'] = args.smtp_server >> + >> + if args.sender is None: >> + error_msg = 'sender' >> + else: >> + config['sender'] = args.sender >> + >> + if args.password is None: >> + error_msg = 'password' >> + else: >> + config['password'] = args.password >> + >> + if args.cc is not None: >> + config['CC'] = args.cc >> + >> + if error_msg is not None: >> + print('Please specify a {} for email output'.format(error_msg)) >> + return None >> + >> + config['output'] = outputs[args.format_output] >> + return config >> + >> +def main(): >> + '''Main entry point''' >> + parser = argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), \ >> + formatter_class=RawTextHelpFormatter) >> + parser.add_argument('--format-output', choices=['terminal','email'], \ >> + default='terminal') >> + parser.add_argument('--smtp-server') >> + parser.add_argument('--password') >> + parser.add_argument('--sender') >> + parser.add_argument('--cc') >> + >> + args = parser.parse_args() >> + config = parse_config(args) >> + if config is None: >> + return >> + >> + symbols = {} >> + lastlib = library = '' >> + >> + output = config['output'](config) >> + >> + for line in sys.stdin: >> + line = line.rstrip('\n') >> + >> + if line.find('mapfile') >= 0: >> + continue >> + library, symbol, name, email = line.split(',') >> + >> + if library != lastlib: >> + message = get_message(lastlib, symbols, config) >> + output.message(message) >> + symbols = {} >> + >> + lastlib = library >> + symbols[symbol] = {'name' : name, 'email' : email} >> + >> + #print the last library >> + message = get_message(lastlib, symbols, config) >> + output.message(message) >> + >> +if __name__ == '__main__': >> + main() >
On 01/09/2021 20:04, Aaron Conole wrote:
> Stephen Hemminger <stephen@networkplumber.org> writes:
>
>> On Wed, 01 Sep 2021 08:31:27 -0400
>> Aaron Conole <aconole@redhat.com> wrote:
>>
>>> $ flake8 ./usertools/dpdk-devbind.py | sed 's@./usertools/dpdk-devbind.py[:0-9]* @@' | sort -u
>>> E128 continuation line under-indented for visual indent
>>> E302 expected 2 blank lines, found 1
>>> E305 expected 2 blank lines after class or function definition, found 1
>>> E501 line too long (105 > 79 characters)
>>> E501 line too long (80 > 79 characters)
>>> E501 line too long (82 > 79 characters)
>>> E501 line too long (83 > 79 characters)
>>> E501 line too long (84 > 79 characters)
>>> E501 line too long (85 > 79 characters)
>>> E501 line too long (86 > 79 characters)
>>> E501 line too long (91 > 79 characters)
>>
>> Current practice on many projects has allowed lines up to 100 characters.
>
> It is probably okay to run with '--ignore=E501' (which will squelch the
> character limit).
I added # noqa : E501 in the appropriate places, as we can't depend on folks using --ignore=E501.
Scripts to count and track the lifecycle of DPDK symbols. The symbol-tool script reports on the growth of symbols over releases and list expired symbols. The notify-symbol-maintainers script consumes the input from symbol-tool and generates email notifications of expired symbols. v2: reworked to fix pylint errors v3: sent with the correct in-reply-to v4: fix typos picked up by the CI v5: fix terminal_size & directory args v6: added list-expired, to list expired experimental symbols v7: fix typo in comments v8: added tool to notify maintainers of expired symbols v9: removed hardcoded emails addressed and script names v10: added ability to identify and notify the original contributors v11: addressed feedback from Aaron Conole, including PEP8 errors. Ray Kinsella (3): devtools: script to track symbols over releases devtools: script to send notifications of expired symbols maintainers: add new abi scripts MAINTAINERS | 2 + devtools/notify-symbol-maintainers.py | 302 +++++++++++++++ devtools/symbol-tool.py | 505 ++++++++++++++++++++++++++ 3 files changed, 809 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py create mode 100755 devtools/symbol-tool.py -- 2.26.2
This script tracks the growth of stable and experimental symbols over releases since v19.11. The script has the ability to count the added symbols between two dpdk releases, and to list experimental symbols present in two dpdk releases (expired symbols). example usages: Count symbols added since v19.11 $ devtools/symbol-tool.py count-symbols Count symbols added since v20.11 $ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05 List experimental symbols present in v20.11 and v21.05 $ devtools/symbol-tool.py list-expired --releases v20.11,v21.05 List experimental symbols in libraries only, present since v19.11 $ devtools/symbol-tool.py list-expired --directory lib Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/symbol-tool.py | 505 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 505 insertions(+) create mode 100755 devtools/symbol-tool.py diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py new file mode 100755 index 0000000000..a0b81c1b90 --- /dev/null +++ b/devtools/symbol-tool.py @@ -0,0 +1,505 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +# pylint: disable=invalid-name +'''Tool to count or list symbols in each DPDK release''' +from pathlib import Path +import sys +import os +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import re +import datetime +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + sys.exit() + +DESCRIPTION = ''' +This script tracks the growth of stable and experimental symbols +over releases since v19.11. The script has the ability to +count the added symbols between two dpdk releases, and to +list experimental symbols present in two dpdk releases +(expired symbols), including the name & email of the original contributor. + +example usages: + +Count symbols added since v19.11 +$ {s} count-symbols + +Count symbols added since v20.11 +$ {s} count-symbols --releases v20.11,v21.05 + +List experimental symbols present in v20.11 and v21.05 +$ {s} list-expired --releases v20.11,v21.05 + +List experimental symbols in libraries only, present since v19.11 +$ {s} list-expired --directory lib +''' + +MAP_GRAMMAR = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" # noqa: E501 + + +def get_abi_versions(): + '''Returns a string of possible dpdk abi versions''' + + year = datetime.date.today().year - 2000 + tags = " |".join(['\'{}\''.format(i) + for i in reversed(range(21, year + 1))]) + tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return tags + + +def get_dpdk_releases(): + '''Returns a list of dpdk release tags names since v19.11''' + + year = datetime.date.today().year - 2000 + year_range = "|".join("{}".format(i) for i in range(19, year + 1)) + pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + try: + result = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + print("Failed to interogate git for release tags") + sys.exit() + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [tag.replace('\"', '') + for tag in reversed(tags) + if pattern.match(tag)][:-3] + + return tags + + +def fix_directory_name(path): + '''Prepend librte to the source directory name''' + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + + +def directory_renamed(path, rel): + '''Fix removal of the librte_ from the directory names''' + + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + try: + result = subprocess.run(['git', 'show', tagfile], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + return result + + +def mapfile_renamed(path, rel): + '''Fix renaming of the map file''' + newfile = None + + result = subprocess.run(['git', 'ls-tree', + rel, str(path.parent) + '/'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if newfile is not None: + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + try: + result = subprocess.run(['git', 'show', tagfile], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + else: + result = None + + return result + + +def mapfile_and_directory_renamed(path, rel): + '''Fix renaming of the map file & the source directory''' + mapfilepath = Path("{}/{}".format(fix_directory_name(path), path.name)) + + return mapfile_renamed(mapfilepath, rel) + + +FIX_STRATEGIES = [directory_renamed, + mapfile_renamed, + mapfile_and_directory_renamed] + + +def get_symbols(map_parser, release, mapfile_path): + '''Count the symbols for a given release and mapfile''' + abi_sections = {} + + tagfile = '{}:{}'.format(release, mapfile_path) + try: + result = subprocess.run(['git', 'show', tagfile], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + for fix_strategy in FIX_STRATEGIES: + if result is not None: + break + result = fix_strategy(mapfile_path, release) + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = map_parser(mapfile).abi() + + return abi_sections + + +def get_terminal_rows(): + '''Find the number of rows in the terminal''' + + try: + return os.get_terminal_size().lines + except IOError: + return 0 + + +class SymbolOwner(): + '''Find the symbols original contributors name and email''' + symbol_regex = {} + blame_regex = {'name': r'author\s(.*)', + 'email': r'author-mail\s<(.*)>'} + + def __init__(self, libpath, symbol): + self.libpath = libpath + self.symbol = symbol + + # find variable definitions in C files, and functions in headers. + self.symbol_regex = \ + {'*.c': r'^(?!extern).*' + self.symbol + '[^()]*;', + '*.h': r'__rte_experimental(?:.*\n){0,2}.*' + self.symbol} + + def find_symbol_location(self): + '''Find where the symbol is definited in the source''' + for key in self.symbol_regex: + for path in Path(self.libpath).rglob(key): + file_text = open(path).read() + + # find where the symbol is defined, either preceeded by + # rte_experimental tag (functions) + # or followed by a ; (variables) + + exp = self.symbol_regex[key] + pattern = re.compile(exp, re.MULTILINE) + search = pattern.search(file_text) + + if search is not None: + symbol_pos = search.span()[1] + symbol_line = file_text.count('\n', 0, symbol_pos) + 1 + + return [str(path), symbol_line] + return None + + def find_symbol_owner(self): + '''Find the symbols original contributors name and email''' + owners = {} + location = self.find_symbol_location() + + if location is None: + return None + + line = '-L {},{}'.format(location[1], location[1]) + # git blame -p(orcelain) -L(ine) path + args = ['-p', line, location[0]] + + try: + result = subprocess.run(['git', 'blame'] + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + return None + + blame = result.stdout.decode('utf-8') + for key in self.blame_regex: + pattern = re.compile(self.blame_regex[key], re.MULTILINE) + match = pattern.search(blame) + + owners[key] = match.groups()[0] + + return owners + + +class SymbolCountOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.OUTPUT_FORMATS[format_output](self, dpdk_releases) + self.column_titles = ['mapfile'] + dpdk_releases + + self.terminal_rows = get_terminal_rows() + self.row = 0 + + def set_terminal_output(self, dpdk_rel): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}' + \ + ''.join(['{:<6}{:<6}'] * (len(dpdk_rel))) + self.column_fmt = '{:50}' + \ + ''.join(['{:<12}'] * (len(dpdk_rel))) + + def set_csv_output(self, dpdk_rel): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},' + \ + ','.join(['{},{}'] * (len(dpdk_rel))) + self.column_fmt = '{},' + \ + ','.join(['{},'] * (len(dpdk_rel))) + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + self.row += 1 + + def print_row(self, mapfile, symbols): + '''Print row of symbol values''' + mapfile = str(mapfile) + print(self.output_fmt.format(*([mapfile] + symbols))) + self.row += 1 + + if((self.terminal_rows > 0) and + ((self.row % self.terminal_rows) == 0)): + self.print_columns() + + OUTPUT_FORMATS = {None: set_terminal_output, + 'terminal': set_terminal_output, + 'csv': set_csv_output} + + +class ListExpiredOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.terminal = True + self.OUTPUT_FORMATS[format_output](self, dpdk_releases) + self.column_titles = ['mapfile'] + \ + ['expired (' + ','.join(dpdk_releases) + ')'] + \ + ['contributor name', 'contributor email'] + + def set_terminal_output(self, _): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}{:<50}{:<25}{:<25}' + self.column_fmt = '{:50}{:50}{:25}{:25}' + + def set_csv_output(self, _): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},{},{},{}' + self.column_fmt = '{},{},{},{}' + self.terminal = False + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + + def print_row(self, mapfile, symbols, owner): + '''Print row of symbol values''' + + for symbol in symbols: + mapfile = str(mapfile) + name = owner[symbol]['name'] \ + if owner[symbol] is not None else '' + email = owner[symbol]['email'] \ + if owner[symbol] is not None else '' + + print(self.output_fmt.format(mapfile, symbol, name, email)) + if self.terminal: + mapfile = '' + + OUTPUT_FORMATS = {None: set_terminal_output, + 'terminal': set_terminal_output, + 'csv': set_csv_output} + + +class CountSymbolsAction: + ''' Logic to count symbols added since a give release ''' + IGNORE_SECTIONS = ['EXPERIMENTAL', 'INTERNAL'] + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.symbols_count = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbol_count = experimental_count = 0 + + symbols = get_symbols(self.parser, release, self.path) + + # which versions are present, and we care about + abi_vers = [abi_ver + for abi_ver in symbols + if abi_ver not in self.IGNORE_SECTIONS] + + for abi_ver in abi_vers: + symbol_count += len(symbols[abi_ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in symbols.keys(): + experimental_count = len(symbols['EXPERIMENTAL']) + + self.symbols_count += [symbol_count, experimental_count] + + def __del__(self): + self.format_output.print_row(self.path.parent, self.symbols_count) + + +class ListExpiredAction: + ''' Logic to list expired symbols between two releases ''' + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.experimental_symbols = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbols = get_symbols(self.parser, release, self.path) + + if 'EXPERIMENTAL' in symbols.keys(): + experimental = [exp.strip() for exp in symbols['EXPERIMENTAL']] + + self.experimental_symbols.append(experimental) + + def __del__(self): + if len(self.experimental_symbols) != 2: + return + + tmp = self.experimental_symbols + # find symbols present in both dpdk releases + intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]] + + # check for empty set + if intersect_syms == []: + return + + sym_owner = {} + for sym in intersect_syms: + sym_owner[sym] = \ + SymbolOwner(self.path.parent, sym).find_symbol_owner() + + self.format_output.print_row(self.path.parent, + intersect_syms, + sym_owner) + + +SRC_DIRECTORIES = 'drivers,lib' + +ACTIONS = {None: CountSymbolsAction, + 'count-symbols': CountSymbolsAction, + 'list-expired': ListExpiredAction} + +ACTION_OUTPUT = {None: SymbolCountOutput, + 'count-symbols': SymbolCountOutput, + 'list-expired': ListExpiredOutput} + + +def main(): + '''Main entry point''' + + dpdk_releases = get_dpdk_releases() + + parser = \ + argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), + formatter_class=RawTextHelpFormatter) + + parser.add_argument('mode', choices=['count-symbols', 'list-expired']) + parser.add_argument('--format-output', choices=['terminal', 'csv'], + default='terminal') + parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','), + default=SRC_DIRECTORIES) + parser.add_argument('--releases', + help='2 x comma separated release tags e.g. \'' + + ','.join([dpdk_releases[0], dpdk_releases[-1]]) + + '\'') + args = parser.parse_args() + + if args.releases is not None: + dpdk_releases = args.releases.split(',') + + if args.mode == 'list-expired': + if len(dpdk_releases) < 2: + sys.exit('Please specify two releases to compare ' + 'in \'list-expired\' mode.') + dpdk_releases = [dpdk_releases[0], + dpdk_releases[len(dpdk_releases) - 1]] + + action = ACTIONS[args.mode] + format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases) + + map_grammar = MAP_GRAMMAR.format(get_abi_versions()) + map_parser = makeGrammar(map_grammar, {}) + + format_output.print_columns() + + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + release_action = action(path, map_parser, format_output) + + for release in dpdk_releases: + release_action.add_mapfile(release) + + # all the magic happens in the destructor + del release_action + + +if __name__ == '__main__': + main() -- 2.26.2
Use this script with the output of the DPDK symbol tool, to notify maintainers of expired symbols by email. You need to define the environment variable DPDK_GETMAINTAINER_PATH for this tool to work. Use terminal output to review the emails before sending. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ devtools/notify_expired_symbols.py --format-output terminal Then use email output to send the emails to the maintainers. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ devtools/notify_expired_symbols.py --format-output email \ --smtp-server <server> --sender <someone@somewhere.com> \ --password <password> --cc <someone@somewhere.com> Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/notify-symbol-maintainers.py | 302 ++++++++++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py new file mode 100755 index 0000000000..edf330f88b --- /dev/null +++ b/devtools/notify-symbol-maintainers.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +# pylint: disable=invalid-name +'''Tool to notify maintainers of expired symbols''' +import os +import smtplib +import ssl +import sys +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import time +from email.message import EmailMessage +from pathlib import Path + +DESCRIPTION = ''' +Use this script with the output of the DPDK symbol tool, to notify maintainers +and contributors of expired symbols by email. You need to define the environment +variable DPDK_GETMAINTAINER_PATH for this tool to work. + +Use terminal output to review the emails before sending. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +{s} --format-output terminal + +Then use email output to send the emails to the maintainers. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +{s} --format-output email \\ +--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\ +--cc <someone@somewhere.com> +''' # noqa: E501 + +EMAIL_TEMPLATE = '''Hi there, + +Please note the symbols listed below have expired. In line with the DPDK ABI +policy, they should be scheduled for removal, in the next DPDK release. + +For more information, please see the DPDK ABI Policy, section 3.5.3. +https://doc.dpdk.org/guides/contributing/abi_policy.html + +Thanks, + +The DPDK Symbol Bot + +''' # noqa: E501 + +ABI_POLICY = 'doc/guides/contributing/abi_policy.rst' +DPDK_GMP_ENV_VAR = 'DPDK_GETMAINTAINER_PATH' +MAINTAINERS = 'MAINTAINERS' +get_maintainer = ['devtools/get-maintainer.sh', + '--email', '-f'] + + +class EnvironException(Exception): + '''Subclass exception for Pylint\'s happiness.''' + + +def _die_on_exception(e): + '''Print an exception, and quit''' + + print('Fatal Error: ' + str(e)) + sys.exit() + + +def _check_get_maintainers_env(): + '''Check get maintainers scripts are setup''' + + if not Path(get_maintainer[0]).is_file(): + raise EnvironException('Cannot locate DPDK\'s get maintainers script, ' + ' usually at $' + get_maintainer[0] + '.') + + if DPDK_GMP_ENV_VAR not in os.environ: + raise EnvironException(DPDK_GMP_ENV_VAR + ' is not defined.') + + if not Path(os.environ[DPDK_GMP_ENV_VAR]).is_file(): + raise EnvironException('Cannot locate get maintainers script, usually' + ' at ' + DPDK_GMP_ENV_VAR + '.') + + +def _get_maintainers(libpath): + '''Get the maintainers for given library''' + + try: + _check_get_maintainers_env() + except EnvironException as e: + _die_on_exception(e) + + try: + cmd = get_maintainer + [libpath] + result = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError as e: + _die_on_exception(e) + + if result is None: + return None + + email = result.stdout.decode('utf-8') + if email == '': + return None + + email = list(filter(None, email.split('\n'))) + return email + + +default_maintainers = _get_maintainers(ABI_POLICY) + \ + _get_maintainers(MAINTAINERS) + + +def get_maintainers(libpath): + '''Get the maintainers for given library''' + maintainers = _get_maintainers(libpath) + + if maintainers is None: + maintainers = default_maintainers + + return maintainers + + +def get_message(library, symbols, config): + '''Build email message from symbols, config and maintainers''' + contributors = {} + message = {} + maintainers = get_maintainers(library) + + if maintainers != default_maintainers: + message['CC'] = default_maintainers.copy() + + if 'CC' in config: + message.setdefault('CC', []).append(config['CC']) + + message['Subject'] = 'Expired symbols in {}\n'.format(library) + + body = EMAIL_TEMPLATE + body += '{:<50}{:<25}{:<25}\n'.format('Symbol', 'Contributor', 'Email') + for sym in symbols: + body += ('{:<50}{:<25}{:<25}\n'.format(sym, + symbols[sym]['name'], + symbols[sym]['email'])) + email = symbols[sym]['email'] + contributors[email] = '' + + contributors = list(contributors.keys()) + + message['To'] = maintainers + contributors + message['Body'] = body + + return message + + +class OutputEmail(): + '''Format the output for email''' + + def __init__(self, config): + self.config = config + + self.terminal = OutputTerminal(config) + context = ssl.create_default_context() + + # Try to log in to server and send email + try: + self.server = smtplib.SMTP(config['smtp_server'], 587) + self.server.starttls(context=context) # Secure the connection + self.server.login(config['sender'], config['password']) + except EnvironException as e: + _die_on_exception(e) + + def message(self, message): + '''send email''' + self.terminal.message(message) + + msg = EmailMessage() + msg.set_content(message.pop('Body')) + + for key in message.keys(): + msg[key] = message[key] + + msg['From'] = self.config['sender'] + msg['Reply-To'] = 'no-reply@dpdk.org' + + self.server.send_message(msg) + + time.sleep(1) + + def __del__(self): + self.server.quit() + + +class OutputTerminal(): # pylint: disable=too-few-public-methods + '''Format the output for the terminal''' + + def __init__(self, config): + self.config = config + + def message(self, message): + '''Print email to terminal''' + + terminal = 'To:' + ', '.join(message['To']) + '\n' + if 'sender' in self.config.keys(): + terminal += 'From:' + self.config['sender'] + '\n' + + terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n' + + if 'CC' in message: + terminal += 'CC:' + ', '.join(message['CC']) + '\n' + + terminal += 'Subject:' + message['Subject'] + '\n' + terminal += 'Body:' + message['Body'] + '\n' + + print(terminal) + print('-' * 80) + + +def parse_config(args): + '''put the command line args in the right places''' + config = {} + error_msg = None + + outputs = { + None: OutputTerminal, + 'terminal': OutputTerminal, + 'email': OutputEmail + } + + if args.format_output == 'email': + if args.smtp_server is None: + error_msg = 'SMTP server' + else: + config['smtp_server'] = args.smtp_server + + if args.sender is None: + error_msg = 'sender' + else: + config['sender'] = args.sender + + if args.password is None: + error_msg = 'password' + else: + config['password'] = args.password + + if args.cc is not None: + config['CC'] = args.cc + + if error_msg is not None: + print('Please specify a {} for email output'.format(error_msg)) + return None + + config['output'] = outputs[args.format_output] + return config + + +def main(): + '''Main entry point''' + parser = \ + argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), + formatter_class=RawTextHelpFormatter) + parser.add_argument('--format-output', + choices=['terminal', 'email'], + default='terminal') + parser.add_argument('--smtp-server') + parser.add_argument('--password') + parser.add_argument('--sender') + parser.add_argument('--cc') + + args = parser.parse_args() + config = parse_config(args) + if config is None: + return + + symbols = {} + lastlib = library = '' + + output = config['output'](config) + + for line in sys.stdin: + line = line.rstrip('\n') + + if line.find('mapfile') >= 0: + continue + library, symbol, name, email = line.split(',') + + if library != lastlib: + message = get_message(lastlib, symbols, config) + output.message(message) + symbols = {} + + lastlib = library + symbols[symbol] = {'name': name, 'email': email} + + # print the last library + message = get_message(lastlib, symbols, config) + output.message(message) + + +if __name__ == '__main__': + main() -- 2.26.2
Add new abi management scripts to the MAINTAINERS file. Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 266f5ac1da..ff8245271f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -129,6 +129,8 @@ F: devtools/gen-abi.sh F: devtools/libabigail.abignore F: devtools/update-abi.sh F: devtools/update_version_map_abi.py +F: devtools/notify-symbol-maintainers.py +F: devtools/symbol-tool.py F: buildtools/check-symbols.sh F: buildtools/map-list-symbol.sh F: drivers/*/*/*.map -- 2.26.2
Hi David, On 01/09/2021 14:01, David Marchand wrote: > Hello Ray, > > On Tue, Aug 31, 2021 at 4:51 PM Ray Kinsella <mdr@ashroe.eu> wrote: >> >> Use this script with the output of the DPDK symbol tool, to notify >> maintainers of expired symbols by email. You need to define the environment >> variable DPDK_GETMAINTAINER_PATH for this tool to work. >> >> Use terminal output to review the emails before sending. Just realized I missed this. > > Two comments: > - there are references of a previous name for the script, > %s/notify_expired_symbols.py/notify-symbol-maintainers.py/g Fixed in v11 = I used __file__ instead. > - and a reminder for the empty report that we received yesterday. > I think this can be reproduced with: Yes - I remember that, I will fix in v12. > > $ DPDK_GETMAINTAINER_PATH=devtools/get_maintainer.pl > devtools/notify-symbol-maintainers.py --format-output terminal <<EOF >> mapfile,expired (v21.08,v19.11),contributor name,contributor email >> lib/rib,rte_rib6_get_ip,Stephen Hemminger,stephen@networkplumber.org >> EOF > To:Ray Kinsella <mdr@ashroe.eu>, Thomas Monjalon <thomas@monjalon.net> > Reply-To:no-reply@dpdk.org > Subject:Expired symbols in > > Body:Hi there, > > Please note the symbols listed below have expired. In line with the DPDK ABI > policy, they should be scheduled for removal, in the next DPDK release. > > For more information, please see the DPDK ABI Policy, section 3.5.3. > https://doc.dpdk.org/guides/contributing/abi_policy.html > > Thanks, > > The DPDK Symbol Bot > > Symbol Contributor > Email > > > -------------------------------------------------------------------------------- > > ^^^^ > Here, empty report. > > To:Vladimir Medvedkin <vladimir.medvedkin@intel.com>, stephen@networkplumber.org > Reply-To:no-reply@dpdk.org > CC:Ray Kinsella <mdr@ashroe.eu>, Thomas Monjalon <thomas@monjalon.net> > Subject:Expired symbols in lib/rib > > Body:Hi there, > > Please note the symbols listed below have expired. In line with the DPDK ABI > policy, they should be scheduled for removal, in the next DPDK release. > > For more information, please see the DPDK ABI Policy, section 3.5.3. > https://doc.dpdk.org/guides/contributing/abi_policy.html > > Thanks, > > The DPDK Symbol Bot > > Symbol Contributor > Email > rte_rib6_get_ip Stephen Hemminger > stephen@networkplumber.org > > > -------------------------------------------------------------------------------- > >
On 01/09/2021 13:46, Aaron Conole wrote:
> Ray Kinsella <mdr@ashroe.eu> writes:
>
>> Use this script with the output of the DPDK symbol tool, to notify
>> maintainers of expired symbols by email. You need to define the environment
>> variable DPDK_GETMAINTAINER_PATH for this tool to work.
>>
>> Use terminal output to review the emails before sending.
>> e.g.
>> $ devtools/symbol-tool.py list-expired --format-output csv \
>> | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
>> devtools/notify_expired_symbols.py --format-output terminal
>>
>> Then use email output to send the emails to the maintainers.
>> e.g.
>> $ devtools/symbol-tool.py list-expired --format-output csv \
>> | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
>> devtools/notify_expired_symbols.py --format-output email \
>> --smtp-server <server> --sender <someone@somewhere.com> \
>> --password <password> --cc <someone@somewhere.com>
>>
>> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
>> ---
>> devtools/notify-symbol-maintainers.py | 256 ++++++++++++++++++++++++++
>> 1 file changed, 256 insertions(+)
>> create mode 100755 devtools/notify-symbol-maintainers.py
>>
>> diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py
>> new file mode 100755
>> index 0000000000..ee554687ff
>> --- /dev/null
>> +++ b/devtools/notify-symbol-maintainers.py
>> @@ -0,0 +1,256 @@
>> +#!/usr/bin/env python3
>> +# SPDX-License-Identifier: BSD-3-Clause
>> +# Copyright(c) 2021 Intel Corporation
>> +# pylint: disable=invalid-name
>> +'''Tool to notify maintainers of expired symbols'''
>> +import smtplib
>> +import ssl
>> +import sys
>> +import subprocess
>> +import argparse
>> +from argparse import RawTextHelpFormatter
>> +import time
>> +from email.message import EmailMessage
>> +
>> +DESCRIPTION = '''
>> +Use this script with the output of the DPDK symbol tool, to notify maintainers
>> +and contributors of expired symbols by email. You need to define the environment
>> +variable DPDK_GETMAINTAINER_PATH for this tool to work.
>> +
>> +Use terminal output to review the emails before sending.
>> +e.g.
>> +$ devtools/symbol-tool.py list-expired --format-output csv \\
>> +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
>> +{s} --format-output terminal
>> +
>> +Then use email output to send the emails to the maintainers.
>> +e.g.
>> +$ devtools/symbol-tool.py list-expired --format-output csv \\
>> +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
>> +{s} --format-output email \\
>> +--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\
>> +--cc <someone@somewhere.com>
>> +'''
>> +
>> +EMAIL_TEMPLATE = '''Hi there,
>> +
>> +Please note the symbols listed below have expired. In line with the DPDK ABI
>> +policy, they should be scheduled for removal, in the next DPDK release.
>> +
>> +For more information, please see the DPDK ABI Policy, section 3.5.3.
>> +https://doc.dpdk.org/guides/contributing/abi_policy.html
>> +
>> +Thanks,
>> +
>> +The DPDK Symbol Bot
>> +
>> +'''
>> +
>> +ABI_POLICY = 'doc/guides/contributing/abi_policy.rst'
>> +MAINTAINERS = 'MAINTAINERS'
>> +get_maintainer = ['devtools/get-maintainer.sh', \
>> + '--email', '-f']
>
> Maybe it's best to make this something that can be overridden. There's
> a series to change the .sh files to .py files. Perhaps an environment
> variable or argument?
>
>> +def _get_maintainers(libpath):
>> + '''Get the maintainers for given library'''
>> + try:
>> + cmd = get_maintainer + [libpath]
>> + result = subprocess.run(cmd, \
>> + stdout=subprocess.PIPE, \
>> + stderr=subprocess.PIPE,
>> + check=True)
>> + except subprocess.CalledProcessError:
>> + return None
>
> You might consider handling
>
> except FileNotFoundError:
> ....
>
> With a graceful exit and error message. In case the get_maintainers
> path changes.
>
So FYI - this get's into the weed a bit.
As there is already a DPDK_GETMAINTAINER_PATH environment variable,
what would you call a new variable.
So instead I added logic for the script to sanity check that _everything_
is defined and where it expects it to be, and then complain loudly
and die when it is not.
The devtools scripts already cross-reference either each, so I'd expect
any changes changing to get-maintainers.sh to get-maintainers.py to take
care of cross-references.
Ray K
On 01/09/2021 14:01, David Marchand wrote:
> Hello Ray,
>
> On Tue, Aug 31, 2021 at 4:51 PM Ray Kinsella <mdr@ashroe.eu> wrote:
>>
>> Use this script with the output of the DPDK symbol tool, to notify
>> maintainers of expired symbols by email. You need to define the environment
>> variable DPDK_GETMAINTAINER_PATH for this tool to work.
>>
>> Use terminal output to review the emails before sending.
>
> Two comments:
> - there are references of a previous name for the script,
> %s/notify_expired_symbols.py/notify-symbol-maintainers.py/g
>
> - and a reminder for the empty report that we received yesterday.
> I think this can be reproduced with:
>
I err'ed - this taken care of v11 also.
Ray K
Scripts to count and track the lifecycle of DPDK symbols. The symbol-tool script reports on the growth of symbols over releases and list expired symbols. The notify-symbol-maintainers script consumes the input from symbol-tool and generates email notifications of expired symbols. v2: reworked to fix pylint errors v3: sent with the correct in-reply-to v4: fix typos picked up by the CI v5: fix terminal_size & directory args v6: added list-expired, to list expired experimental symbols v7: fix typo in comments v8: added tool to notify maintainers of expired symbols v9: removed hardcoded emails addressed and script names v10: added ability to identify and notify the original contributors v11: addressed feedback from Aaron Conole, including PEP8 errors. Ray Kinsella (3): devtools: script to track symbols over releases devtools: script to send notifications of expired symbols maintainers: add new abi scripts MAINTAINERS | 2 + devtools/notify-symbol-maintainers.py | 302 +++++++++++++++ devtools/symbol-tool.py | 505 ++++++++++++++++++++++++++ 3 files changed, 809 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py create mode 100755 devtools/symbol-tool.py -- 2.26.2
This script tracks the growth of stable and experimental symbols over releases since v19.11. The script has the ability to count the added symbols between two dpdk releases, and to list experimental symbols present in two dpdk releases (expired symbols). example usages: Count symbols added since v19.11 $ devtools/symbol-tool.py count-symbols Count symbols added since v20.11 $ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05 List experimental symbols present in v20.11 and v21.05 $ devtools/symbol-tool.py list-expired --releases v20.11,v21.05 List experimental symbols in libraries only, present since v19.11 $ devtools/symbol-tool.py list-expired --directory lib Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/symbol-tool.py | 505 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 505 insertions(+) create mode 100755 devtools/symbol-tool.py diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py new file mode 100755 index 0000000000..a0b81c1b90 --- /dev/null +++ b/devtools/symbol-tool.py @@ -0,0 +1,505 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +# pylint: disable=invalid-name +'''Tool to count or list symbols in each DPDK release''' +from pathlib import Path +import sys +import os +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import re +import datetime +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + sys.exit() + +DESCRIPTION = ''' +This script tracks the growth of stable and experimental symbols +over releases since v19.11. The script has the ability to +count the added symbols between two dpdk releases, and to +list experimental symbols present in two dpdk releases +(expired symbols), including the name & email of the original contributor. + +example usages: + +Count symbols added since v19.11 +$ {s} count-symbols + +Count symbols added since v20.11 +$ {s} count-symbols --releases v20.11,v21.05 + +List experimental symbols present in v20.11 and v21.05 +$ {s} list-expired --releases v20.11,v21.05 + +List experimental symbols in libraries only, present since v19.11 +$ {s} list-expired --directory lib +''' + +MAP_GRAMMAR = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" # noqa: E501 + + +def get_abi_versions(): + '''Returns a string of possible dpdk abi versions''' + + year = datetime.date.today().year - 2000 + tags = " |".join(['\'{}\''.format(i) + for i in reversed(range(21, year + 1))]) + tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return tags + + +def get_dpdk_releases(): + '''Returns a list of dpdk release tags names since v19.11''' + + year = datetime.date.today().year - 2000 + year_range = "|".join("{}".format(i) for i in range(19, year + 1)) + pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + try: + result = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + print("Failed to interogate git for release tags") + sys.exit() + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [tag.replace('\"', '') + for tag in reversed(tags) + if pattern.match(tag)][:-3] + + return tags + + +def fix_directory_name(path): + '''Prepend librte to the source directory name''' + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + + +def directory_renamed(path, rel): + '''Fix removal of the librte_ from the directory names''' + + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + try: + result = subprocess.run(['git', 'show', tagfile], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + return result + + +def mapfile_renamed(path, rel): + '''Fix renaming of the map file''' + newfile = None + + result = subprocess.run(['git', 'ls-tree', + rel, str(path.parent) + '/'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if newfile is not None: + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + try: + result = subprocess.run(['git', 'show', tagfile], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + else: + result = None + + return result + + +def mapfile_and_directory_renamed(path, rel): + '''Fix renaming of the map file & the source directory''' + mapfilepath = Path("{}/{}".format(fix_directory_name(path), path.name)) + + return mapfile_renamed(mapfilepath, rel) + + +FIX_STRATEGIES = [directory_renamed, + mapfile_renamed, + mapfile_and_directory_renamed] + + +def get_symbols(map_parser, release, mapfile_path): + '''Count the symbols for a given release and mapfile''' + abi_sections = {} + + tagfile = '{}:{}'.format(release, mapfile_path) + try: + result = subprocess.run(['git', 'show', tagfile], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + for fix_strategy in FIX_STRATEGIES: + if result is not None: + break + result = fix_strategy(mapfile_path, release) + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = map_parser(mapfile).abi() + + return abi_sections + + +def get_terminal_rows(): + '''Find the number of rows in the terminal''' + + try: + return os.get_terminal_size().lines + except IOError: + return 0 + + +class SymbolOwner(): + '''Find the symbols original contributors name and email''' + symbol_regex = {} + blame_regex = {'name': r'author\s(.*)', + 'email': r'author-mail\s<(.*)>'} + + def __init__(self, libpath, symbol): + self.libpath = libpath + self.symbol = symbol + + # find variable definitions in C files, and functions in headers. + self.symbol_regex = \ + {'*.c': r'^(?!extern).*' + self.symbol + '[^()]*;', + '*.h': r'__rte_experimental(?:.*\n){0,2}.*' + self.symbol} + + def find_symbol_location(self): + '''Find where the symbol is definited in the source''' + for key in self.symbol_regex: + for path in Path(self.libpath).rglob(key): + file_text = open(path).read() + + # find where the symbol is defined, either preceeded by + # rte_experimental tag (functions) + # or followed by a ; (variables) + + exp = self.symbol_regex[key] + pattern = re.compile(exp, re.MULTILINE) + search = pattern.search(file_text) + + if search is not None: + symbol_pos = search.span()[1] + symbol_line = file_text.count('\n', 0, symbol_pos) + 1 + + return [str(path), symbol_line] + return None + + def find_symbol_owner(self): + '''Find the symbols original contributors name and email''' + owners = {} + location = self.find_symbol_location() + + if location is None: + return None + + line = '-L {},{}'.format(location[1], location[1]) + # git blame -p(orcelain) -L(ine) path + args = ['-p', line, location[0]] + + try: + result = subprocess.run(['git', 'blame'] + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + return None + + blame = result.stdout.decode('utf-8') + for key in self.blame_regex: + pattern = re.compile(self.blame_regex[key], re.MULTILINE) + match = pattern.search(blame) + + owners[key] = match.groups()[0] + + return owners + + +class SymbolCountOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.OUTPUT_FORMATS[format_output](self, dpdk_releases) + self.column_titles = ['mapfile'] + dpdk_releases + + self.terminal_rows = get_terminal_rows() + self.row = 0 + + def set_terminal_output(self, dpdk_rel): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}' + \ + ''.join(['{:<6}{:<6}'] * (len(dpdk_rel))) + self.column_fmt = '{:50}' + \ + ''.join(['{:<12}'] * (len(dpdk_rel))) + + def set_csv_output(self, dpdk_rel): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},' + \ + ','.join(['{},{}'] * (len(dpdk_rel))) + self.column_fmt = '{},' + \ + ','.join(['{},'] * (len(dpdk_rel))) + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + self.row += 1 + + def print_row(self, mapfile, symbols): + '''Print row of symbol values''' + mapfile = str(mapfile) + print(self.output_fmt.format(*([mapfile] + symbols))) + self.row += 1 + + if((self.terminal_rows > 0) and + ((self.row % self.terminal_rows) == 0)): + self.print_columns() + + OUTPUT_FORMATS = {None: set_terminal_output, + 'terminal': set_terminal_output, + 'csv': set_csv_output} + + +class ListExpiredOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.terminal = True + self.OUTPUT_FORMATS[format_output](self, dpdk_releases) + self.column_titles = ['mapfile'] + \ + ['expired (' + ','.join(dpdk_releases) + ')'] + \ + ['contributor name', 'contributor email'] + + def set_terminal_output(self, _): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}{:<50}{:<25}{:<25}' + self.column_fmt = '{:50}{:50}{:25}{:25}' + + def set_csv_output(self, _): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},{},{},{}' + self.column_fmt = '{},{},{},{}' + self.terminal = False + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + + def print_row(self, mapfile, symbols, owner): + '''Print row of symbol values''' + + for symbol in symbols: + mapfile = str(mapfile) + name = owner[symbol]['name'] \ + if owner[symbol] is not None else '' + email = owner[symbol]['email'] \ + if owner[symbol] is not None else '' + + print(self.output_fmt.format(mapfile, symbol, name, email)) + if self.terminal: + mapfile = '' + + OUTPUT_FORMATS = {None: set_terminal_output, + 'terminal': set_terminal_output, + 'csv': set_csv_output} + + +class CountSymbolsAction: + ''' Logic to count symbols added since a give release ''' + IGNORE_SECTIONS = ['EXPERIMENTAL', 'INTERNAL'] + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.symbols_count = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbol_count = experimental_count = 0 + + symbols = get_symbols(self.parser, release, self.path) + + # which versions are present, and we care about + abi_vers = [abi_ver + for abi_ver in symbols + if abi_ver not in self.IGNORE_SECTIONS] + + for abi_ver in abi_vers: + symbol_count += len(symbols[abi_ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in symbols.keys(): + experimental_count = len(symbols['EXPERIMENTAL']) + + self.symbols_count += [symbol_count, experimental_count] + + def __del__(self): + self.format_output.print_row(self.path.parent, self.symbols_count) + + +class ListExpiredAction: + ''' Logic to list expired symbols between two releases ''' + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.experimental_symbols = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbols = get_symbols(self.parser, release, self.path) + + if 'EXPERIMENTAL' in symbols.keys(): + experimental = [exp.strip() for exp in symbols['EXPERIMENTAL']] + + self.experimental_symbols.append(experimental) + + def __del__(self): + if len(self.experimental_symbols) != 2: + return + + tmp = self.experimental_symbols + # find symbols present in both dpdk releases + intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]] + + # check for empty set + if intersect_syms == []: + return + + sym_owner = {} + for sym in intersect_syms: + sym_owner[sym] = \ + SymbolOwner(self.path.parent, sym).find_symbol_owner() + + self.format_output.print_row(self.path.parent, + intersect_syms, + sym_owner) + + +SRC_DIRECTORIES = 'drivers,lib' + +ACTIONS = {None: CountSymbolsAction, + 'count-symbols': CountSymbolsAction, + 'list-expired': ListExpiredAction} + +ACTION_OUTPUT = {None: SymbolCountOutput, + 'count-symbols': SymbolCountOutput, + 'list-expired': ListExpiredOutput} + + +def main(): + '''Main entry point''' + + dpdk_releases = get_dpdk_releases() + + parser = \ + argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), + formatter_class=RawTextHelpFormatter) + + parser.add_argument('mode', choices=['count-symbols', 'list-expired']) + parser.add_argument('--format-output', choices=['terminal', 'csv'], + default='terminal') + parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','), + default=SRC_DIRECTORIES) + parser.add_argument('--releases', + help='2 x comma separated release tags e.g. \'' + + ','.join([dpdk_releases[0], dpdk_releases[-1]]) + + '\'') + args = parser.parse_args() + + if args.releases is not None: + dpdk_releases = args.releases.split(',') + + if args.mode == 'list-expired': + if len(dpdk_releases) < 2: + sys.exit('Please specify two releases to compare ' + 'in \'list-expired\' mode.') + dpdk_releases = [dpdk_releases[0], + dpdk_releases[len(dpdk_releases) - 1]] + + action = ACTIONS[args.mode] + format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases) + + map_grammar = MAP_GRAMMAR.format(get_abi_versions()) + map_parser = makeGrammar(map_grammar, {}) + + format_output.print_columns() + + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + release_action = action(path, map_parser, format_output) + + for release in dpdk_releases: + release_action.add_mapfile(release) + + # all the magic happens in the destructor + del release_action + + +if __name__ == '__main__': + main() -- 2.26.2
Use this script with the output of the DPDK symbol tool, to notify maintainers of expired symbols by email. You need to define the environment variable DPDK_GETMAINTAINER_PATH for this tool to work. Use terminal output to review the emails before sending. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ devtools/notify_expired_symbols.py --format-output terminal Then use email output to send the emails to the maintainers. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ devtools/notify_expired_symbols.py --format-output email \ --smtp-server <server> --sender <someone@somewhere.com> \ --password <password> --cc <someone@somewhere.com> Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/notify-symbol-maintainers.py | 302 ++++++++++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py new file mode 100755 index 0000000000..edf330f88b --- /dev/null +++ b/devtools/notify-symbol-maintainers.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +# pylint: disable=invalid-name +'''Tool to notify maintainers of expired symbols''' +import os +import smtplib +import ssl +import sys +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import time +from email.message import EmailMessage +from pathlib import Path + +DESCRIPTION = ''' +Use this script with the output of the DPDK symbol tool, to notify maintainers +and contributors of expired symbols by email. You need to define the environment +variable DPDK_GETMAINTAINER_PATH for this tool to work. + +Use terminal output to review the emails before sending. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +{s} --format-output terminal + +Then use email output to send the emails to the maintainers. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +{s} --format-output email \\ +--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\ +--cc <someone@somewhere.com> +''' # noqa: E501 + +EMAIL_TEMPLATE = '''Hi there, + +Please note the symbols listed below have expired. In line with the DPDK ABI +policy, they should be scheduled for removal, in the next DPDK release. + +For more information, please see the DPDK ABI Policy, section 3.5.3. +https://doc.dpdk.org/guides/contributing/abi_policy.html + +Thanks, + +The DPDK Symbol Bot + +''' # noqa: E501 + +ABI_POLICY = 'doc/guides/contributing/abi_policy.rst' +DPDK_GMP_ENV_VAR = 'DPDK_GETMAINTAINER_PATH' +MAINTAINERS = 'MAINTAINERS' +get_maintainer = ['devtools/get-maintainer.sh', + '--email', '-f'] + + +class EnvironException(Exception): + '''Subclass exception for Pylint\'s happiness.''' + + +def _die_on_exception(e): + '''Print an exception, and quit''' + + print('Fatal Error: ' + str(e)) + sys.exit() + + +def _check_get_maintainers_env(): + '''Check get maintainers scripts are setup''' + + if not Path(get_maintainer[0]).is_file(): + raise EnvironException('Cannot locate DPDK\'s get maintainers script, ' + ' usually at $' + get_maintainer[0] + '.') + + if DPDK_GMP_ENV_VAR not in os.environ: + raise EnvironException(DPDK_GMP_ENV_VAR + ' is not defined.') + + if not Path(os.environ[DPDK_GMP_ENV_VAR]).is_file(): + raise EnvironException('Cannot locate get maintainers script, usually' + ' at ' + DPDK_GMP_ENV_VAR + '.') + + +def _get_maintainers(libpath): + '''Get the maintainers for given library''' + + try: + _check_get_maintainers_env() + except EnvironException as e: + _die_on_exception(e) + + try: + cmd = get_maintainer + [libpath] + result = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError as e: + _die_on_exception(e) + + if result is None: + return None + + email = result.stdout.decode('utf-8') + if email == '': + return None + + email = list(filter(None, email.split('\n'))) + return email + + +default_maintainers = _get_maintainers(ABI_POLICY) + \ + _get_maintainers(MAINTAINERS) + + +def get_maintainers(libpath): + '''Get the maintainers for given library''' + maintainers = _get_maintainers(libpath) + + if maintainers is None: + maintainers = default_maintainers + + return maintainers + + +def get_message(library, symbols, config): + '''Build email message from symbols, config and maintainers''' + contributors = {} + message = {} + maintainers = get_maintainers(library) + + if maintainers != default_maintainers: + message['CC'] = default_maintainers.copy() + + if 'CC' in config: + message.setdefault('CC', []).append(config['CC']) + + message['Subject'] = 'Expired symbols in {}\n'.format(library) + + body = EMAIL_TEMPLATE + body += '{:<50}{:<25}{:<25}\n'.format('Symbol', 'Contributor', 'Email') + for sym in symbols: + body += ('{:<50}{:<25}{:<25}\n'.format(sym, + symbols[sym]['name'], + symbols[sym]['email'])) + email = symbols[sym]['email'] + contributors[email] = '' + + contributors = list(contributors.keys()) + + message['To'] = maintainers + contributors + message['Body'] = body + + return message + + +class OutputEmail(): + '''Format the output for email''' + + def __init__(self, config): + self.config = config + + self.terminal = OutputTerminal(config) + context = ssl.create_default_context() + + # Try to log in to server and send email + try: + self.server = smtplib.SMTP(config['smtp_server'], 587) + self.server.starttls(context=context) # Secure the connection + self.server.login(config['sender'], config['password']) + except EnvironException as e: + _die_on_exception(e) + + def message(self, message): + '''send email''' + self.terminal.message(message) + + msg = EmailMessage() + msg.set_content(message.pop('Body')) + + for key in message.keys(): + msg[key] = message[key] + + msg['From'] = self.config['sender'] + msg['Reply-To'] = 'no-reply@dpdk.org' + + self.server.send_message(msg) + + time.sleep(1) + + def __del__(self): + self.server.quit() + + +class OutputTerminal(): # pylint: disable=too-few-public-methods + '''Format the output for the terminal''' + + def __init__(self, config): + self.config = config + + def message(self, message): + '''Print email to terminal''' + + terminal = 'To:' + ', '.join(message['To']) + '\n' + if 'sender' in self.config.keys(): + terminal += 'From:' + self.config['sender'] + '\n' + + terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n' + + if 'CC' in message: + terminal += 'CC:' + ', '.join(message['CC']) + '\n' + + terminal += 'Subject:' + message['Subject'] + '\n' + terminal += 'Body:' + message['Body'] + '\n' + + print(terminal) + print('-' * 80) + + +def parse_config(args): + '''put the command line args in the right places''' + config = {} + error_msg = None + + outputs = { + None: OutputTerminal, + 'terminal': OutputTerminal, + 'email': OutputEmail + } + + if args.format_output == 'email': + if args.smtp_server is None: + error_msg = 'SMTP server' + else: + config['smtp_server'] = args.smtp_server + + if args.sender is None: + error_msg = 'sender' + else: + config['sender'] = args.sender + + if args.password is None: + error_msg = 'password' + else: + config['password'] = args.password + + if args.cc is not None: + config['CC'] = args.cc + + if error_msg is not None: + print('Please specify a {} for email output'.format(error_msg)) + return None + + config['output'] = outputs[args.format_output] + return config + + +def main(): + '''Main entry point''' + parser = \ + argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), + formatter_class=RawTextHelpFormatter) + parser.add_argument('--format-output', + choices=['terminal', 'email'], + default='terminal') + parser.add_argument('--smtp-server') + parser.add_argument('--password') + parser.add_argument('--sender') + parser.add_argument('--cc') + + args = parser.parse_args() + config = parse_config(args) + if config is None: + return + + symbols = {} + lastlib = library = '' + + output = config['output'](config) + + for line in sys.stdin: + line = line.rstrip('\n') + + if line.find('mapfile') >= 0: + continue + library, symbol, name, email = line.split(',') + + if library != lastlib: + message = get_message(lastlib, symbols, config) + output.message(message) + symbols = {} + + lastlib = library + symbols[symbol] = {'name': name, 'email': email} + + # print the last library + message = get_message(lastlib, symbols, config) + output.message(message) + + +if __name__ == '__main__': + main() -- 2.26.2
Add new abi management scripts to the MAINTAINERS file. Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 266f5ac1da..ff8245271f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -129,6 +129,8 @@ F: devtools/gen-abi.sh F: devtools/libabigail.abignore F: devtools/update-abi.sh F: devtools/update_version_map_abi.py +F: devtools/notify-symbol-maintainers.py +F: devtools/symbol-tool.py F: buildtools/check-symbols.sh F: buildtools/map-list-symbol.sh F: drivers/*/*/*.map -- 2.26.2
Scripts to count and track the lifecycle of DPDK symbols. The symbol-tool script reports on the growth of symbols over releases and list expired symbols. The notify-symbol-maintainers script consumes the input from symbol-tool and generates email notifications of expired symbols. v2: reworked to fix pylint errors v3: sent with the correct in-reply-to v4: fix typos picked up by the CI v5: fix terminal_size & directory args v6: added list-expired, to list expired experimental symbols v7: fix typo in comments v8: added tool to notify maintainers of expired symbols v9: removed hardcoded emails addressed and script names v10: added ability to identify and notify the original contributors v11: addressed feedback from Aaron Conole, including PEP8 errors. v12: added symbol-tool ignore functionality, to ignore specific symbols. Ray Kinsella (4): devtools: script to track symbols over releases devtools: script to send notifications of expired symbols maintainers: add new abi scripts devtools: add asym crypto to symbol-tool ignore MAINTAINERS | 2 + devtools/notify-symbol-maintainers.py | 302 ++++++++++++++ devtools/symbol-tool.py | 566 ++++++++++++++++++++++++++ devtools/symboltool.ignore | 3 + 4 files changed, 873 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py create mode 100755 devtools/symbol-tool.py create mode 100644 devtools/symboltool.ignore -- 2.26.2
This script tracks the growth of stable and experimental symbols over releases since v19.11. The script has the ability to count the added symbols between two dpdk releases, and to list experimental symbols present in two dpdk releases (expired symbols). example usages: Count symbols added since v19.11 $ devtools/symbol-tool.py count-symbols Count symbols added since v20.11 $ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05 List experimental symbols present in v20.11 and v21.05 $ devtools/symbol-tool.py list-expired --releases v20.11,v21.05 List experimental symbols in libraries only, present since v19.11 $ devtools/symbol-tool.py list-expired --directory lib Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/symbol-tool.py | 566 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 566 insertions(+) create mode 100755 devtools/symbol-tool.py diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py new file mode 100755 index 0000000000..a71ab59539 --- /dev/null +++ b/devtools/symbol-tool.py @@ -0,0 +1,566 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +# pylint: disable=invalid-name +'''Tool to count or list symbols in each DPDK release''' +from pathlib import Path +import sys +import os +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import re +import datetime +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + sys.exit() + +DESCRIPTION = ''' +This script tracks the growth of stable and experimental symbols +over releases since v19.11. The script has the ability to +count the added symbols between two dpdk releases, and to +list experimental symbols present in two dpdk releases +(expired symbols), including the name & email of the original contributor. + +example usages: + +Count symbols added since v19.11 +$ {s} count-symbols + +Count symbols added since v20.11 +$ {s} count-symbols --releases v20.11,v21.05 + +List experimental symbols present in v20.11 and v21.05 +$ {s} list-expired --releases v20.11,v21.05 + +List experimental symbols in libraries only, present since v19.11 +$ {s} list-expired --directory lib +''' + +MAP_GRAMMAR = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" # noqa: E501 + + +class EnvironException(Exception): + '''Subclass exception for Pylint\'s happiness.''' + + +def get_abi_versions(): + '''Returns a string of possible dpdk abi versions''' + + year = datetime.date.today().year - 2000 + tags = " |".join(['\'{}\''.format(i) + for i in reversed(range(21, year + 1))]) + tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return tags + + +def get_dpdk_releases(): + '''Returns a list of dpdk release tags names since v19.11''' + + year = datetime.date.today().year - 2000 + year_range = "|".join("{}".format(i) for i in range(19, year + 1)) + pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + try: + result = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + print("Failed to interogate git for release tags") + sys.exit() + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [tag.replace('\"', '') + for tag in reversed(tags) + if pattern.match(tag)][:-3] + + return tags + + +def fix_directory_name(path): + '''Prepend librte to the source directory name''' + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + + +def directory_renamed(path, rel): + '''Fix removal of the librte_ from the directory names''' + + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + try: + result = subprocess.run(['git', 'show', tagfile], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + return result + + +def mapfile_renamed(path, rel): + '''Fix renaming of the map file''' + newfile = None + + result = subprocess.run(['git', 'ls-tree', + rel, str(path.parent) + '/'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if newfile is not None: + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + try: + result = subprocess.run(['git', 'show', tagfile], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + else: + result = None + + return result + + +def mapfile_and_directory_renamed(path, rel): + '''Fix renaming of the map file & the source directory''' + mapfilepath = Path("{}/{}".format(fix_directory_name(path), path.name)) + + return mapfile_renamed(mapfilepath, rel) + + +FIX_STRATEGIES = [directory_renamed, + mapfile_renamed, + mapfile_and_directory_renamed] + + +def get_symbols(map_parser, release, mapfile_path): + '''Count the symbols for a given release and mapfile''' + abi_sections = {} + + tagfile = '{}:{}'.format(release, mapfile_path) + try: + result = subprocess.run(['git', 'show', tagfile], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + for fix_strategy in FIX_STRATEGIES: + if result is not None: + break + result = fix_strategy(mapfile_path, release) + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = map_parser(mapfile).abi() + + return abi_sections + + +def get_terminal_rows(): + '''Find the number of rows in the terminal''' + + try: + return os.get_terminal_size().lines + except IOError: + return 0 + + +class IgnoredSymbols(): # pylint: disable=too-few-public-methods + '''Symbols which are to be be ignored for some period''' + + SYMBOL_TOOL_IGNORE = 'devtools/symboltool.ignore' + ignore_regex = [] + __initialized = False + + @staticmethod + def initialize(): + '''intialize once''' + + if IgnoredSymbols.__initialized: + return + IgnoredSymbols.__initialized = True + + if 'DPDK_SYMBOL_TOOL_IGNORE' in os.environ: + IgnoredSymbols.SYMBOL_TOOL_IGNORE = \ + os.environ['DPDK_SYMBOL_TOOL_IGNORE'] + + # if the user specifies an ignore file, we can't find then error. + if not Path(IgnoredSymbols.SYMBOL_TOOL_IGNORE).is_file(): + raise EnvironException('Cannot locate {}\'s ' + 'ignore file'.format(__file__)) + + # if we cannot find the default ignore file, then continue + if not Path(IgnoredSymbols.SYMBOL_TOOL_IGNORE).is_file(): + return + + lines = open(Path(IgnoredSymbols.SYMBOL_TOOL_IGNORE)).readlines() + for line in lines: + + line = line.strip() + + # ignore comments and whitespace + if line.startswith(';') or len(line) == 0: + continue + + IgnoredSymbols.ignore_regex.append(re.compile(line)) + + def __init__(self): + self.initialize() + + def check_ignore(self, symbol): + '''Check symbol against the ignore regexes''' + + for exp in self.ignore_regex: + if exp.search(symbol) is not None: + return True + + return False + + +class SymbolOwner(): + '''Find the symbols original contributors name and email''' + symbol_regex = {} + blame_regex = {'name': r'author\s(.*)', + 'email': r'author-mail\s<(.*)>'} + + def __init__(self, libpath, symbol): + self.libpath = libpath + self.symbol = symbol + + # find variable definitions in C files, and functions in headers. + self.symbol_regex = \ + {'*.c': r'^(?!extern).*' + self.symbol + '[^()]*;', + '*.h': r'__rte_experimental(?:.*\n){0,2}.*' + self.symbol} + + def find_symbol_location(self): + '''Find where the symbol is definited in the source''' + for key in self.symbol_regex: + for path in Path(self.libpath).rglob(key): + file_text = open(path).read() + + # find where the symbol is defined, either preceded by + # rte_experimental tag (functions) + # or followed by a ; (variables) + + exp = self.symbol_regex[key] + pattern = re.compile(exp, re.MULTILINE) + search = pattern.search(file_text) + + if search is not None: + symbol_pos = search.span()[1] + symbol_line = file_text.count('\n', 0, symbol_pos) + 1 + + return [str(path), symbol_line] + return None + + def find_symbol_owner(self): + '''Find the symbols original contributors name and email''' + owners = {} + location = self.find_symbol_location() + + if location is None: + return None + + line = '-L {},{}'.format(location[1], location[1]) + # git blame -p(orcelain) -L(ine) path + args = ['-p', line, location[0]] + + try: + result = subprocess.run(['git', 'blame'] + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + return None + + blame = result.stdout.decode('utf-8') + for key in self.blame_regex: + pattern = re.compile(self.blame_regex[key], re.MULTILINE) + match = pattern.search(blame) + + owners[key] = match.groups()[0] + + return owners + + +class SymbolCountOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.OUTPUT_FORMATS[format_output](self, dpdk_releases) + self.column_titles = ['mapfile'] + dpdk_releases + + self.terminal_rows = get_terminal_rows() + self.row = 0 + + def set_terminal_output(self, dpdk_rel): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}' + \ + ''.join(['{:<6}{:<6}'] * (len(dpdk_rel))) + self.column_fmt = '{:50}' + \ + ''.join(['{:<12}'] * (len(dpdk_rel))) + + def set_csv_output(self, dpdk_rel): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},' + \ + ','.join(['{},{}'] * (len(dpdk_rel))) + self.column_fmt = '{},' + \ + ','.join(['{},'] * (len(dpdk_rel))) + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + self.row += 1 + + def print_row(self, mapfile, symbols): + '''Print row of symbol values''' + mapfile = str(mapfile) + print(self.output_fmt.format(*([mapfile] + symbols))) + self.row += 1 + + if((self.terminal_rows > 0) and + ((self.row % self.terminal_rows) == 0)): + self.print_columns() + + OUTPUT_FORMATS = {None: set_terminal_output, + 'terminal': set_terminal_output, + 'csv': set_csv_output} + + +class ListExpiredOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.terminal = True + self.OUTPUT_FORMATS[format_output](self, dpdk_releases) + self.column_titles = ['mapfile'] + \ + ['expired (' + ','.join(dpdk_releases) + ')'] + \ + ['contributor name', 'contributor email'] + + def set_terminal_output(self, _): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}{:<50}{:<25}{:<25}' + self.column_fmt = '{:50}{:50}{:25}{:25}' + + def set_csv_output(self, _): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},{},{},{}' + self.column_fmt = '{},{},{},{}' + self.terminal = False + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + + def print_row(self, mapfile, symbols, owner): + '''Print row of symbol values''' + + for symbol in symbols: + mapfile = str(mapfile) + name = owner[symbol]['name'] \ + if owner[symbol] is not None else '' + email = owner[symbol]['email'] \ + if owner[symbol] is not None else '' + + print(self.output_fmt.format(mapfile, symbol, name, email)) + if self.terminal: + mapfile = '' + + OUTPUT_FORMATS = {None: set_terminal_output, + 'terminal': set_terminal_output, + 'csv': set_csv_output} + + +class CountSymbolsAction: + ''' Logic to count symbols added since a give release ''' + IGNORE_SECTIONS = ['EXPERIMENTAL', 'INTERNAL'] + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.symbols_count = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbol_count = experimental_count = 0 + + symbols = get_symbols(self.parser, release, self.path) + + # which versions are present, and we care about + abi_vers = [abi_ver + for abi_ver in symbols + if abi_ver not in self.IGNORE_SECTIONS] + + for abi_ver in abi_vers: + symbol_count += len(symbols[abi_ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in symbols.keys(): + experimental_count = len(symbols['EXPERIMENTAL']) + + self.symbols_count += [symbol_count, experimental_count] + + def __del__(self): + self.format_output.print_row(self.path.parent, self.symbols_count) + + +class ListExpiredAction: + ''' Logic to list expired symbols between two releases ''' + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.experimental_symbols = [] + self.ignored_symbols = IgnoredSymbols() + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbols = get_symbols(self.parser, release, self.path) + + if 'EXPERIMENTAL' in symbols.keys(): + experimental = [exp.strip() for exp in symbols['EXPERIMENTAL']] + + self.experimental_symbols.append(experimental) + + def __del__(self): + if len(self.experimental_symbols) != 2: + return + + tmp = self.experimental_symbols + # find symbols present in both dpdk releases + intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]] + + # remove ignored symbols + intersect_syms = [sym for sym in intersect_syms if not + self.ignored_symbols.check_ignore(sym)] + + # check for empty set + if intersect_syms == []: + return + + sym_owner = {} + for sym in intersect_syms: + sym_owner[sym] = \ + SymbolOwner(self.path.parent, sym).find_symbol_owner() + + self.format_output.print_row(self.path.parent, + intersect_syms, + sym_owner) + + +SRC_DIRECTORIES = 'drivers,lib' + +ACTIONS = {None: CountSymbolsAction, + 'count-symbols': CountSymbolsAction, + 'list-expired': ListExpiredAction} + +ACTION_OUTPUT = {None: SymbolCountOutput, + 'count-symbols': SymbolCountOutput, + 'list-expired': ListExpiredOutput} + + +def main(): + '''Main entry point''' + + dpdk_releases = get_dpdk_releases() + + parser = \ + argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), + formatter_class=RawTextHelpFormatter) + + parser.add_argument('mode', choices=['count-symbols', 'list-expired']) + parser.add_argument('--format-output', choices=['terminal', 'csv'], + default='terminal') + parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','), + default=SRC_DIRECTORIES) + parser.add_argument('--releases', + help='2 x comma separated release tags e.g. \'' + + ','.join([dpdk_releases[0], dpdk_releases[-1]]) + + '\'') + args = parser.parse_args() + + if args.releases is not None: + dpdk_releases = args.releases.split(',') + + if args.mode == 'list-expired': + if len(dpdk_releases) < 2: + sys.exit('Please specify two releases to compare ' + 'in \'list-expired\' mode.') + dpdk_releases = [dpdk_releases[0], + dpdk_releases[len(dpdk_releases) - 1]] + + action = ACTIONS[args.mode] + format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases) + + map_grammar = MAP_GRAMMAR.format(get_abi_versions()) + map_parser = makeGrammar(map_grammar, {}) + + format_output.print_columns() + + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + release_action = action(path, map_parser, format_output) + + for release in dpdk_releases: + release_action.add_mapfile(release) + + # all the magic happens in the destructor + del release_action + + +if __name__ == '__main__': + main() -- 2.26.2
Use this script with the output of the DPDK symbol tool, to notify maintainers of expired symbols by email. You need to define the environment variable DPDK_GETMAINTAINER_PATH for this tool to work. Use terminal output to review the emails before sending. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ devtools/notify_expired_symbols.py --format-output terminal Then use email output to send the emails to the maintainers. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ devtools/notify_expired_symbols.py --format-output email \ --smtp-server <server> --sender <someone@somewhere.com> \ --password <password> --cc <someone@somewhere.com> Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/notify-symbol-maintainers.py | 302 ++++++++++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py new file mode 100755 index 0000000000..edf330f88b --- /dev/null +++ b/devtools/notify-symbol-maintainers.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +# pylint: disable=invalid-name +'''Tool to notify maintainers of expired symbols''' +import os +import smtplib +import ssl +import sys +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import time +from email.message import EmailMessage +from pathlib import Path + +DESCRIPTION = ''' +Use this script with the output of the DPDK symbol tool, to notify maintainers +and contributors of expired symbols by email. You need to define the environment +variable DPDK_GETMAINTAINER_PATH for this tool to work. + +Use terminal output to review the emails before sending. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +{s} --format-output terminal + +Then use email output to send the emails to the maintainers. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +{s} --format-output email \\ +--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\ +--cc <someone@somewhere.com> +''' # noqa: E501 + +EMAIL_TEMPLATE = '''Hi there, + +Please note the symbols listed below have expired. In line with the DPDK ABI +policy, they should be scheduled for removal, in the next DPDK release. + +For more information, please see the DPDK ABI Policy, section 3.5.3. +https://doc.dpdk.org/guides/contributing/abi_policy.html + +Thanks, + +The DPDK Symbol Bot + +''' # noqa: E501 + +ABI_POLICY = 'doc/guides/contributing/abi_policy.rst' +DPDK_GMP_ENV_VAR = 'DPDK_GETMAINTAINER_PATH' +MAINTAINERS = 'MAINTAINERS' +get_maintainer = ['devtools/get-maintainer.sh', + '--email', '-f'] + + +class EnvironException(Exception): + '''Subclass exception for Pylint\'s happiness.''' + + +def _die_on_exception(e): + '''Print an exception, and quit''' + + print('Fatal Error: ' + str(e)) + sys.exit() + + +def _check_get_maintainers_env(): + '''Check get maintainers scripts are setup''' + + if not Path(get_maintainer[0]).is_file(): + raise EnvironException('Cannot locate DPDK\'s get maintainers script, ' + ' usually at $' + get_maintainer[0] + '.') + + if DPDK_GMP_ENV_VAR not in os.environ: + raise EnvironException(DPDK_GMP_ENV_VAR + ' is not defined.') + + if not Path(os.environ[DPDK_GMP_ENV_VAR]).is_file(): + raise EnvironException('Cannot locate get maintainers script, usually' + ' at ' + DPDK_GMP_ENV_VAR + '.') + + +def _get_maintainers(libpath): + '''Get the maintainers for given library''' + + try: + _check_get_maintainers_env() + except EnvironException as e: + _die_on_exception(e) + + try: + cmd = get_maintainer + [libpath] + result = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError as e: + _die_on_exception(e) + + if result is None: + return None + + email = result.stdout.decode('utf-8') + if email == '': + return None + + email = list(filter(None, email.split('\n'))) + return email + + +default_maintainers = _get_maintainers(ABI_POLICY) + \ + _get_maintainers(MAINTAINERS) + + +def get_maintainers(libpath): + '''Get the maintainers for given library''' + maintainers = _get_maintainers(libpath) + + if maintainers is None: + maintainers = default_maintainers + + return maintainers + + +def get_message(library, symbols, config): + '''Build email message from symbols, config and maintainers''' + contributors = {} + message = {} + maintainers = get_maintainers(library) + + if maintainers != default_maintainers: + message['CC'] = default_maintainers.copy() + + if 'CC' in config: + message.setdefault('CC', []).append(config['CC']) + + message['Subject'] = 'Expired symbols in {}\n'.format(library) + + body = EMAIL_TEMPLATE + body += '{:<50}{:<25}{:<25}\n'.format('Symbol', 'Contributor', 'Email') + for sym in symbols: + body += ('{:<50}{:<25}{:<25}\n'.format(sym, + symbols[sym]['name'], + symbols[sym]['email'])) + email = symbols[sym]['email'] + contributors[email] = '' + + contributors = list(contributors.keys()) + + message['To'] = maintainers + contributors + message['Body'] = body + + return message + + +class OutputEmail(): + '''Format the output for email''' + + def __init__(self, config): + self.config = config + + self.terminal = OutputTerminal(config) + context = ssl.create_default_context() + + # Try to log in to server and send email + try: + self.server = smtplib.SMTP(config['smtp_server'], 587) + self.server.starttls(context=context) # Secure the connection + self.server.login(config['sender'], config['password']) + except EnvironException as e: + _die_on_exception(e) + + def message(self, message): + '''send email''' + self.terminal.message(message) + + msg = EmailMessage() + msg.set_content(message.pop('Body')) + + for key in message.keys(): + msg[key] = message[key] + + msg['From'] = self.config['sender'] + msg['Reply-To'] = 'no-reply@dpdk.org' + + self.server.send_message(msg) + + time.sleep(1) + + def __del__(self): + self.server.quit() + + +class OutputTerminal(): # pylint: disable=too-few-public-methods + '''Format the output for the terminal''' + + def __init__(self, config): + self.config = config + + def message(self, message): + '''Print email to terminal''' + + terminal = 'To:' + ', '.join(message['To']) + '\n' + if 'sender' in self.config.keys(): + terminal += 'From:' + self.config['sender'] + '\n' + + terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n' + + if 'CC' in message: + terminal += 'CC:' + ', '.join(message['CC']) + '\n' + + terminal += 'Subject:' + message['Subject'] + '\n' + terminal += 'Body:' + message['Body'] + '\n' + + print(terminal) + print('-' * 80) + + +def parse_config(args): + '''put the command line args in the right places''' + config = {} + error_msg = None + + outputs = { + None: OutputTerminal, + 'terminal': OutputTerminal, + 'email': OutputEmail + } + + if args.format_output == 'email': + if args.smtp_server is None: + error_msg = 'SMTP server' + else: + config['smtp_server'] = args.smtp_server + + if args.sender is None: + error_msg = 'sender' + else: + config['sender'] = args.sender + + if args.password is None: + error_msg = 'password' + else: + config['password'] = args.password + + if args.cc is not None: + config['CC'] = args.cc + + if error_msg is not None: + print('Please specify a {} for email output'.format(error_msg)) + return None + + config['output'] = outputs[args.format_output] + return config + + +def main(): + '''Main entry point''' + parser = \ + argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), + formatter_class=RawTextHelpFormatter) + parser.add_argument('--format-output', + choices=['terminal', 'email'], + default='terminal') + parser.add_argument('--smtp-server') + parser.add_argument('--password') + parser.add_argument('--sender') + parser.add_argument('--cc') + + args = parser.parse_args() + config = parse_config(args) + if config is None: + return + + symbols = {} + lastlib = library = '' + + output = config['output'](config) + + for line in sys.stdin: + line = line.rstrip('\n') + + if line.find('mapfile') >= 0: + continue + library, symbol, name, email = line.split(',') + + if library != lastlib: + message = get_message(lastlib, symbols, config) + output.message(message) + symbols = {} + + lastlib = library + symbols[symbol] = {'name': name, 'email': email} + + # print the last library + message = get_message(lastlib, symbols, config) + output.message(message) + + +if __name__ == '__main__': + main() -- 2.26.2
Add new abi management scripts to the MAINTAINERS file. Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 266f5ac1da..ff8245271f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -129,6 +129,8 @@ F: devtools/gen-abi.sh F: devtools/libabigail.abignore F: devtools/update-abi.sh F: devtools/update_version_map_abi.py +F: devtools/notify-symbol-maintainers.py +F: devtools/symbol-tool.py F: buildtools/check-symbols.sh F: buildtools/map-list-symbol.sh F: drivers/*/*/*.map -- 2.26.2
Add asym crypto to the symbol-tool's ignore file. Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/symboltool.ignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 devtools/symboltool.ignore diff --git a/devtools/symboltool.ignore b/devtools/symboltool.ignore new file mode 100644 index 0000000000..800c500a82 --- /dev/null +++ b/devtools/symboltool.ignore @@ -0,0 +1,3 @@ +; regex of symbols for the symbol-tool to ignore + +rte_cryptodev_asym_.* -- 2.26.2
> ----------------------------------------------------------------------
> Add asym crypto to the symbol-tool's ignore file.
>
> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
> ---
Acked-by: Akhil Goyal <gakhil@marvell.com>
Scripts to count and track the lifecycle of DPDK symbols. The symbol-tool script reports on the growth of symbols over releases and list expired symbols. The notify-symbol-maintainers script consumes the input from symbol-tool and generates email notifications of expired symbols. v2: reworked to fix pylint errors v3: sent with the correct in-reply-to v4: fix typos picked up by the CI v5: fix terminal_size & directory args v6: added list-expired, to list expired experimental symbols v7: fix typo in comments v8: added tool to notify maintainers of expired symbols v9: removed hardcoded emails addressed and script names v10: added ability to identify and notify the original contributors v11: addressed feedback from Aaron Conole, including PEP8 errors. v12: added symbol-tool ignore functionality, to ignore specific symbols v13: renamed symboltool.abignore, typos, added ack from Akhil Goyal Ray Kinsella (4): devtools: script to track symbols over releases devtools: script to send notifications of expired symbols maintainers: add new abi scripts devtools: add asym crypto to symbol-tool ignore MAINTAINERS | 3 + devtools/notify-symbol-maintainers.py | 302 ++++++++++++++ devtools/symbol-tool.py | 566 ++++++++++++++++++++++++++ devtools/symboltool.abignore | 3 + 4 files changed, 874 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py create mode 100755 devtools/symbol-tool.py create mode 100644 devtools/symboltool.abignore -- 2.26.2
This script tracks the growth of stable and experimental symbols over releases since v19.11. The script has the ability to count the added symbols between two dpdk releases, and to list experimental symbols present in two dpdk releases (expired symbols). example usages: Count symbols added since v19.11 $ devtools/symbol-tool.py count-symbols Count symbols added since v20.11 $ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05 List experimental symbols present in v20.11 and v21.05 $ devtools/symbol-tool.py list-expired --releases v20.11,v21.05 List experimental symbols in libraries only, present since v19.11 $ devtools/symbol-tool.py list-expired --directory lib Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/symbol-tool.py | 566 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 566 insertions(+) create mode 100755 devtools/symbol-tool.py diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py new file mode 100755 index 0000000000..a77d8b2442 --- /dev/null +++ b/devtools/symbol-tool.py @@ -0,0 +1,566 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +# pylint: disable=invalid-name +'''Tool to count or list symbols in each DPDK release''' +from pathlib import Path +import sys +import os +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import re +import datetime +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + sys.exit() + +DESCRIPTION = ''' +This script tracks the growth of stable and experimental symbols +over releases since v19.11. The script has the ability to +count the added symbols between two dpdk releases, and to +list experimental symbols present in two dpdk releases +(expired symbols), including the name & email of the original contributor. + +example usages: + +Count symbols added since v19.11 +$ {s} count-symbols + +Count symbols added since v20.11 +$ {s} count-symbols --releases v20.11,v21.05 + +List experimental symbols present in v20.11 and v21.05 +$ {s} list-expired --releases v20.11,v21.05 + +List experimental symbols in libraries only, present since v19.11 +$ {s} list-expired --directory lib +''' + +MAP_GRAMMAR = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" # noqa: E501 + + +class EnvironException(Exception): + '''Subclass exception for Pylint\'s happiness.''' + + +def get_abi_versions(): + '''Returns a string of possible dpdk abi versions''' + + year = datetime.date.today().year - 2000 + tags = " |".join(['\'{}\''.format(i) + for i in reversed(range(21, year + 1))]) + tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return tags + + +def get_dpdk_releases(): + '''Returns a list of dpdk release tags names since v19.11''' + + year = datetime.date.today().year - 2000 + year_range = "|".join("{}".format(i) for i in range(19, year + 1)) + pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + try: + result = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + print("Failed to interogate git for release tags") + sys.exit() + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [tag.replace('\"', '') + for tag in reversed(tags) + if pattern.match(tag)][:-3] + + return tags + + +def fix_directory_name(path): + '''Prepend librte to the source directory name''' + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + + +def directory_renamed(path, rel): + '''Fix removal of the librte_ from the directory names''' + + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + try: + result = subprocess.run(['git', 'show', tagfile], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + return result + + +def mapfile_renamed(path, rel): + '''Fix renaming of the map file''' + newfile = None + + result = subprocess.run(['git', 'ls-tree', + rel, str(path.parent) + '/'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if newfile is not None: + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + try: + result = subprocess.run(['git', 'show', tagfile], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + else: + result = None + + return result + + +def mapfile_and_directory_renamed(path, rel): + '''Fix renaming of the map file & the source directory''' + mapfilepath = Path("{}/{}".format(fix_directory_name(path), path.name)) + + return mapfile_renamed(mapfilepath, rel) + + +FIX_STRATEGIES = [directory_renamed, + mapfile_renamed, + mapfile_and_directory_renamed] + + +def get_symbols(map_parser, release, mapfile_path): + '''Count the symbols for a given release and mapfile''' + abi_sections = {} + + tagfile = '{}:{}'.format(release, mapfile_path) + try: + result = subprocess.run(['git', 'show', tagfile], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + result = None + + for fix_strategy in FIX_STRATEGIES: + if result is not None: + break + result = fix_strategy(mapfile_path, release) + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = map_parser(mapfile).abi() + + return abi_sections + + +def get_terminal_rows(): + '''Find the number of rows in the terminal''' + + try: + return os.get_terminal_size().lines + except IOError: + return 0 + + +class IgnoredSymbols(): # pylint: disable=too-few-public-methods + '''Symbols which are to be ignored for some period''' + + SYMBOL_TOOL_IGNORE = 'devtools/symboltool.abignore' + ignore_regex = [] + __initialized = False + + @staticmethod + def initialize(): + '''initialize once''' + + if IgnoredSymbols.__initialized: + return + IgnoredSymbols.__initialized = True + + if 'DPDK_SYMBOL_TOOL_IGNORE' in os.environ: + IgnoredSymbols.SYMBOL_TOOL_IGNORE = \ + os.environ['DPDK_SYMBOL_TOOL_IGNORE'] + + # if the user specifies an ignore file, we can't find then error. + if not Path(IgnoredSymbols.SYMBOL_TOOL_IGNORE).is_file(): + raise EnvironException('Cannot locate {}\'s ' + 'ignore file'.format(__file__)) + + # if we cannot find the default ignore file, then continue + if not Path(IgnoredSymbols.SYMBOL_TOOL_IGNORE).is_file(): + return + + lines = open(Path(IgnoredSymbols.SYMBOL_TOOL_IGNORE)).readlines() + for line in lines: + + line = line.strip() + + # ignore comments and whitespace + if line.startswith(';') or len(line) == 0: + continue + + IgnoredSymbols.ignore_regex.append(re.compile(line)) + + def __init__(self): + self.initialize() + + def check_ignore(self, symbol): + '''Check symbol against the ignore regexes''' + + for exp in self.ignore_regex: + if exp.search(symbol) is not None: + return True + + return False + + +class SymbolOwner(): + '''Find the symbols original contributors name and email''' + symbol_regex = {} + blame_regex = {'name': r'author\s(.*)', + 'email': r'author-mail\s<(.*)>'} + + def __init__(self, libpath, symbol): + self.libpath = libpath + self.symbol = symbol + + # find variable definitions in C files, and functions in headers. + self.symbol_regex = \ + {'*.c': r'^(?!extern).*' + self.symbol + '[^()]*;', + '*.h': r'__rte_experimental(?:.*\n){0,2}.*' + self.symbol} + + def find_symbol_location(self): + '''Find where the symbol is definited in the source''' + for key in self.symbol_regex: + for path in Path(self.libpath).rglob(key): + file_text = open(path).read() + + # find where the symbol is defined, either preceded by + # rte_experimental tag (functions) + # or followed by a ; (variables) + + exp = self.symbol_regex[key] + pattern = re.compile(exp, re.MULTILINE) + search = pattern.search(file_text) + + if search is not None: + symbol_pos = search.span()[1] + symbol_line = file_text.count('\n', 0, symbol_pos) + 1 + + return [str(path), symbol_line] + return None + + def find_symbol_owner(self): + '''Find the symbols original contributors name and email''' + owners = {} + location = self.find_symbol_location() + + if location is None: + return None + + line = '-L {},{}'.format(location[1], location[1]) + # git blame -p(orcelain) -L(ine) path + args = ['-p', line, location[0]] + + try: + result = subprocess.run(['git', 'blame'] + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError: + return None + + blame = result.stdout.decode('utf-8') + for key in self.blame_regex: + pattern = re.compile(self.blame_regex[key], re.MULTILINE) + match = pattern.search(blame) + + owners[key] = match.groups()[0] + + return owners + + +class SymbolCountOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.OUTPUT_FORMATS[format_output](self, dpdk_releases) + self.column_titles = ['mapfile'] + dpdk_releases + + self.terminal_rows = get_terminal_rows() + self.row = 0 + + def set_terminal_output(self, dpdk_rel): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}' + \ + ''.join(['{:<6}{:<6}'] * (len(dpdk_rel))) + self.column_fmt = '{:50}' + \ + ''.join(['{:<12}'] * (len(dpdk_rel))) + + def set_csv_output(self, dpdk_rel): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},' + \ + ','.join(['{},{}'] * (len(dpdk_rel))) + self.column_fmt = '{},' + \ + ','.join(['{},'] * (len(dpdk_rel))) + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + self.row += 1 + + def print_row(self, mapfile, symbols): + '''Print row of symbol values''' + mapfile = str(mapfile) + print(self.output_fmt.format(*([mapfile] + symbols))) + self.row += 1 + + if((self.terminal_rows > 0) and + ((self.row % self.terminal_rows) == 0)): + self.print_columns() + + OUTPUT_FORMATS = {None: set_terminal_output, + 'terminal': set_terminal_output, + 'csv': set_csv_output} + + +class ListExpiredOutput(): + '''Format the output to supported formats''' + output_fmt = "" + column_fmt = "" + + def __init__(self, format_output, dpdk_releases): + self.terminal = True + self.OUTPUT_FORMATS[format_output](self, dpdk_releases) + self.column_titles = ['mapfile'] + \ + ['expired (' + ','.join(dpdk_releases) + ')'] + \ + ['contributor name', 'contributor email'] + + def set_terminal_output(self, _): + '''Set the output format to Tabbed Separated Values''' + + self.output_fmt = '{:<50}{:<50}{:<25}{:<25}' + self.column_fmt = '{:50}{:50}{:25}{:25}' + + def set_csv_output(self, _): + '''Set the output format to Comma Separated Values''' + + self.output_fmt = '{},{},{},{}' + self.column_fmt = '{},{},{},{}' + self.terminal = False + + def print_columns(self): + '''Print column rows with release names''' + print(self.column_fmt.format(*self.column_titles)) + + def print_row(self, mapfile, symbols, owner): + '''Print row of symbol values''' + + for symbol in symbols: + mapfile = str(mapfile) + name = owner[symbol]['name'] \ + if owner[symbol] is not None else '' + email = owner[symbol]['email'] \ + if owner[symbol] is not None else '' + + print(self.output_fmt.format(mapfile, symbol, name, email)) + if self.terminal: + mapfile = '' + + OUTPUT_FORMATS = {None: set_terminal_output, + 'terminal': set_terminal_output, + 'csv': set_csv_output} + + +class CountSymbolsAction: + ''' Logic to count symbols added since a give release ''' + IGNORE_SECTIONS = ['EXPERIMENTAL', 'INTERNAL'] + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.symbols_count = [] + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbol_count = experimental_count = 0 + + symbols = get_symbols(self.parser, release, self.path) + + # which versions are present, and we care about + abi_vers = [abi_ver + for abi_ver in symbols + if abi_ver not in self.IGNORE_SECTIONS] + + for abi_ver in abi_vers: + symbol_count += len(symbols[abi_ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in symbols.keys(): + experimental_count = len(symbols['EXPERIMENTAL']) + + self.symbols_count += [symbol_count, experimental_count] + + def __del__(self): + self.format_output.print_row(self.path.parent, self.symbols_count) + + +class ListExpiredAction: + ''' Logic to list expired symbols between two releases ''' + + def __init__(self, mapfile_path, mapfile_parser, format_output): + self.path = mapfile_path + self.parser = mapfile_parser + self.format_output = format_output + self.experimental_symbols = [] + self.ignored_symbols = IgnoredSymbols() + + def add_mapfile(self, release): + ''' add a version mapfile ''' + symbols = get_symbols(self.parser, release, self.path) + + if 'EXPERIMENTAL' in symbols.keys(): + experimental = [exp.strip() for exp in symbols['EXPERIMENTAL']] + + self.experimental_symbols.append(experimental) + + def __del__(self): + if len(self.experimental_symbols) != 2: + return + + tmp = self.experimental_symbols + # find symbols present in both dpdk releases + intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]] + + # remove ignored symbols + intersect_syms = [sym for sym in intersect_syms if not + self.ignored_symbols.check_ignore(sym)] + + # check for empty set + if intersect_syms == []: + return + + sym_owner = {} + for sym in intersect_syms: + sym_owner[sym] = \ + SymbolOwner(self.path.parent, sym).find_symbol_owner() + + self.format_output.print_row(self.path.parent, + intersect_syms, + sym_owner) + + +SRC_DIRECTORIES = 'drivers,lib' + +ACTIONS = {None: CountSymbolsAction, + 'count-symbols': CountSymbolsAction, + 'list-expired': ListExpiredAction} + +ACTION_OUTPUT = {None: SymbolCountOutput, + 'count-symbols': SymbolCountOutput, + 'list-expired': ListExpiredOutput} + + +def main(): + '''Main entry point''' + + dpdk_releases = get_dpdk_releases() + + parser = \ + argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), + formatter_class=RawTextHelpFormatter) + + parser.add_argument('mode', choices=['count-symbols', 'list-expired']) + parser.add_argument('--format-output', choices=['terminal', 'csv'], + default='terminal') + parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','), + default=SRC_DIRECTORIES) + parser.add_argument('--releases', + help='2 x comma separated release tags e.g. \'' + + ','.join([dpdk_releases[0], dpdk_releases[-1]]) + + '\'') + args = parser.parse_args() + + if args.releases is not None: + dpdk_releases = args.releases.split(',') + + if args.mode == 'list-expired': + if len(dpdk_releases) < 2: + sys.exit('Please specify two releases to compare ' + 'in \'list-expired\' mode.') + dpdk_releases = [dpdk_releases[0], + dpdk_releases[len(dpdk_releases) - 1]] + + action = ACTIONS[args.mode] + format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases) + + map_grammar = MAP_GRAMMAR.format(get_abi_versions()) + map_parser = makeGrammar(map_grammar, {}) + + format_output.print_columns() + + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + release_action = action(path, map_parser, format_output) + + for release in dpdk_releases: + release_action.add_mapfile(release) + + # all the magic happens in the destructor + del release_action + + +if __name__ == '__main__': + main() -- 2.26.2
Use this script with the output of the DPDK symbol tool, to notify maintainers of expired symbols by email. You need to define the environment variable DPDK_GETMAINTAINER_PATH for this tool to work. Use terminal output to review the emails before sending. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ devtools/notify_expired_symbols.py --format-output terminal Then use email output to send the emails to the maintainers. e.g. $ devtools/symbol-tool.py list-expired --format-output csv \ | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \ devtools/notify_expired_symbols.py --format-output email \ --smtp-server <server> --sender <someone@somewhere.com> \ --password <password> --cc <someone@somewhere.com> Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- devtools/notify-symbol-maintainers.py | 302 ++++++++++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100755 devtools/notify-symbol-maintainers.py diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py new file mode 100755 index 0000000000..edf330f88b --- /dev/null +++ b/devtools/notify-symbol-maintainers.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +# pylint: disable=invalid-name +'''Tool to notify maintainers of expired symbols''' +import os +import smtplib +import ssl +import sys +import subprocess +import argparse +from argparse import RawTextHelpFormatter +import time +from email.message import EmailMessage +from pathlib import Path + +DESCRIPTION = ''' +Use this script with the output of the DPDK symbol tool, to notify maintainers +and contributors of expired symbols by email. You need to define the environment +variable DPDK_GETMAINTAINER_PATH for this tool to work. + +Use terminal output to review the emails before sending. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +{s} --format-output terminal + +Then use email output to send the emails to the maintainers. +e.g. +$ devtools/symbol-tool.py list-expired --format-output csv \\ +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\ +{s} --format-output email \\ +--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\ +--cc <someone@somewhere.com> +''' # noqa: E501 + +EMAIL_TEMPLATE = '''Hi there, + +Please note the symbols listed below have expired. In line with the DPDK ABI +policy, they should be scheduled for removal, in the next DPDK release. + +For more information, please see the DPDK ABI Policy, section 3.5.3. +https://doc.dpdk.org/guides/contributing/abi_policy.html + +Thanks, + +The DPDK Symbol Bot + +''' # noqa: E501 + +ABI_POLICY = 'doc/guides/contributing/abi_policy.rst' +DPDK_GMP_ENV_VAR = 'DPDK_GETMAINTAINER_PATH' +MAINTAINERS = 'MAINTAINERS' +get_maintainer = ['devtools/get-maintainer.sh', + '--email', '-f'] + + +class EnvironException(Exception): + '''Subclass exception for Pylint\'s happiness.''' + + +def _die_on_exception(e): + '''Print an exception, and quit''' + + print('Fatal Error: ' + str(e)) + sys.exit() + + +def _check_get_maintainers_env(): + '''Check get maintainers scripts are setup''' + + if not Path(get_maintainer[0]).is_file(): + raise EnvironException('Cannot locate DPDK\'s get maintainers script, ' + ' usually at $' + get_maintainer[0] + '.') + + if DPDK_GMP_ENV_VAR not in os.environ: + raise EnvironException(DPDK_GMP_ENV_VAR + ' is not defined.') + + if not Path(os.environ[DPDK_GMP_ENV_VAR]).is_file(): + raise EnvironException('Cannot locate get maintainers script, usually' + ' at ' + DPDK_GMP_ENV_VAR + '.') + + +def _get_maintainers(libpath): + '''Get the maintainers for given library''' + + try: + _check_get_maintainers_env() + except EnvironException as e: + _die_on_exception(e) + + try: + cmd = get_maintainer + [libpath] + result = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + except subprocess.CalledProcessError as e: + _die_on_exception(e) + + if result is None: + return None + + email = result.stdout.decode('utf-8') + if email == '': + return None + + email = list(filter(None, email.split('\n'))) + return email + + +default_maintainers = _get_maintainers(ABI_POLICY) + \ + _get_maintainers(MAINTAINERS) + + +def get_maintainers(libpath): + '''Get the maintainers for given library''' + maintainers = _get_maintainers(libpath) + + if maintainers is None: + maintainers = default_maintainers + + return maintainers + + +def get_message(library, symbols, config): + '''Build email message from symbols, config and maintainers''' + contributors = {} + message = {} + maintainers = get_maintainers(library) + + if maintainers != default_maintainers: + message['CC'] = default_maintainers.copy() + + if 'CC' in config: + message.setdefault('CC', []).append(config['CC']) + + message['Subject'] = 'Expired symbols in {}\n'.format(library) + + body = EMAIL_TEMPLATE + body += '{:<50}{:<25}{:<25}\n'.format('Symbol', 'Contributor', 'Email') + for sym in symbols: + body += ('{:<50}{:<25}{:<25}\n'.format(sym, + symbols[sym]['name'], + symbols[sym]['email'])) + email = symbols[sym]['email'] + contributors[email] = '' + + contributors = list(contributors.keys()) + + message['To'] = maintainers + contributors + message['Body'] = body + + return message + + +class OutputEmail(): + '''Format the output for email''' + + def __init__(self, config): + self.config = config + + self.terminal = OutputTerminal(config) + context = ssl.create_default_context() + + # Try to log in to server and send email + try: + self.server = smtplib.SMTP(config['smtp_server'], 587) + self.server.starttls(context=context) # Secure the connection + self.server.login(config['sender'], config['password']) + except EnvironException as e: + _die_on_exception(e) + + def message(self, message): + '''send email''' + self.terminal.message(message) + + msg = EmailMessage() + msg.set_content(message.pop('Body')) + + for key in message.keys(): + msg[key] = message[key] + + msg['From'] = self.config['sender'] + msg['Reply-To'] = 'no-reply@dpdk.org' + + self.server.send_message(msg) + + time.sleep(1) + + def __del__(self): + self.server.quit() + + +class OutputTerminal(): # pylint: disable=too-few-public-methods + '''Format the output for the terminal''' + + def __init__(self, config): + self.config = config + + def message(self, message): + '''Print email to terminal''' + + terminal = 'To:' + ', '.join(message['To']) + '\n' + if 'sender' in self.config.keys(): + terminal += 'From:' + self.config['sender'] + '\n' + + terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n' + + if 'CC' in message: + terminal += 'CC:' + ', '.join(message['CC']) + '\n' + + terminal += 'Subject:' + message['Subject'] + '\n' + terminal += 'Body:' + message['Body'] + '\n' + + print(terminal) + print('-' * 80) + + +def parse_config(args): + '''put the command line args in the right places''' + config = {} + error_msg = None + + outputs = { + None: OutputTerminal, + 'terminal': OutputTerminal, + 'email': OutputEmail + } + + if args.format_output == 'email': + if args.smtp_server is None: + error_msg = 'SMTP server' + else: + config['smtp_server'] = args.smtp_server + + if args.sender is None: + error_msg = 'sender' + else: + config['sender'] = args.sender + + if args.password is None: + error_msg = 'password' + else: + config['password'] = args.password + + if args.cc is not None: + config['CC'] = args.cc + + if error_msg is not None: + print('Please specify a {} for email output'.format(error_msg)) + return None + + config['output'] = outputs[args.format_output] + return config + + +def main(): + '''Main entry point''' + parser = \ + argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), + formatter_class=RawTextHelpFormatter) + parser.add_argument('--format-output', + choices=['terminal', 'email'], + default='terminal') + parser.add_argument('--smtp-server') + parser.add_argument('--password') + parser.add_argument('--sender') + parser.add_argument('--cc') + + args = parser.parse_args() + config = parse_config(args) + if config is None: + return + + symbols = {} + lastlib = library = '' + + output = config['output'](config) + + for line in sys.stdin: + line = line.rstrip('\n') + + if line.find('mapfile') >= 0: + continue + library, symbol, name, email = line.split(',') + + if library != lastlib: + message = get_message(lastlib, symbols, config) + output.message(message) + symbols = {} + + lastlib = library + symbols[symbol] = {'name': name, 'email': email} + + # print the last library + message = get_message(lastlib, symbols, config) + output.message(message) + + +if __name__ == '__main__': + main() -- 2.26.2
Add new abi management scripts to the MAINTAINERS file. Signed-off-by: Ray Kinsella <mdr@ashroe.eu> --- MAINTAINERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 266f5ac1da..ae38af1b85 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -127,8 +127,11 @@ F: devtools/check-abi-version.sh F: devtools/check-symbol-change.sh F: devtools/gen-abi.sh F: devtools/libabigail.abignore +F: devtools/symboltool.abignore F: devtools/update-abi.sh F: devtools/update_version_map_abi.py +F: devtools/notify-symbol-maintainers.py +F: devtools/symbol-tool.py F: buildtools/check-symbols.sh F: buildtools/map-list-symbol.sh F: drivers/*/*/*.map -- 2.26.2
Add asym crypto to the symbol-tool's ignore file. Signed-off-by: Ray Kinsella <mdr@ashroe.eu> Acked-by: Akhil Goyal <gakhil@marvell.com> --- devtools/symboltool.abignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 devtools/symboltool.abignore diff --git a/devtools/symboltool.abignore b/devtools/symboltool.abignore new file mode 100644 index 0000000000..800c500a82 --- /dev/null +++ b/devtools/symboltool.abignore @@ -0,0 +1,3 @@ +; regex of symbols for the symbol-tool to ignore + +rte_cryptodev_asym_.* -- 2.26.2
On Thu, 9 Sep 2021 14:48:04 +0100
Ray Kinsella <mdr@ashroe.eu> wrote:
> The symbol-tool script reports on the growth of symbols over releases
> and list expired symbols. The notify-symbol-maintainers script
> consumes the input from symbol-tool and generates email notifications
> of expired symbols.
>
> v2: reworked to fix pylint errors
> v3: sent with the correct in-reply-to
> v4: fix typos picked up by the CI
> v5: fix terminal_size & directory args
> v6: added list-expired, to list expired experimental symbols
> v7: fix typo in comments
> v8: added tool to notify maintainers of expired symbols
> v9: removed hardcoded emails addressed and script names
> v10: added ability to identify and notify the original contributors
> v11: addressed feedback from Aaron Conole, including PEP8 errors.
> v12: added symbol-tool ignore functionality, to ignore specific symbols
> v13: renamed symboltool.abignore, typos, added ack from Akhil Goyal
>
> Ray Kinsella (4):
> devtools: script to track symbols over releases
> devtools: script to send notifications of expired symbols
> maintainers: add new abi scripts
> devtools: add asym crypto to symbol-tool ignore
Not sure why this never made it in.
Series-Acked-by: Stephen Hemminger <stephen@networkplumber.org>