* [dpdk-dev] [PATCH] devtools: script to track map symbols
@ 2021-06-18 16:36 Ray Kinsella
2021-06-18 19:40 ` Stephen Hemminger
` (12 more replies)
0 siblings, 13 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-06-18 16:36 UTC (permalink / raw)
To: dev; +Cc: ferruh.yigit, thomas, ktraynor, bruce.richardson, mdr
Script to track growth of stable and experimental symbols
over releases since v19.11.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/count_symbols.py | 230 ++++++++++++++++++++++++++++++++++++++
1 file changed, 230 insertions(+)
create mode 100755 devtools/count_symbols.py
diff --git a/devtools/count_symbols.py b/devtools/count_symbols.py
new file mode 100755
index 0000000000..7b29651044
--- /dev/null
+++ b/devtools/count_symbols.py
@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+from pathlib import Path
+import sys, os
+import subprocess
+import argparse
+import re
+import datetime
+
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ exit()
+
+symbolMapGrammar = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+"""
+
+#abi_ver = ['21', '20.0.1', '20.0', '20']
+
+def get_abi_versions():
+ year = datetime.date.today().year - 2000
+ s=" |".join(['\'{}\''.format(i) for i in reversed(range(21, year + 1)) ])
+ s = s + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return s
+
+def get_dpdk_releases():
+ year = datetime.date.today().year - 2000
+ s="|".join("{}".format(i) for i in range(19,year + 1))
+ pattern = re.compile('^\"v(' + s + ')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE)
+ if result.stderr.startswith(b'fatal'):
+ result = None
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [ tag.replace('\"','') \
+ for tag in reversed(tags) \
+ if pattern.match(tag) ][:-3]
+
+ return tags
+
+
+def get_terminal_rows():
+ rows, _ = os.popen('stty size', 'r').read().split()
+ return int(rows)
+
+def fix_directory_name(path):
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+# fix removal of the librte_ from the directory names
+def directory_renamed(path, rel):
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE)
+ if result.stderr.startswith(b'fatal'):
+ result = None
+
+ return result
+
+# fix renaming of map files
+def mapfile_renamed(path, rel):
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree', \
+ rel, str(path.parent) + '/'], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if(newfile is not None):
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE)
+ if result.stderr.startswith(b'fatal'):
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+# renaming of the map file & renaming of directory
+def mapfile_and_directory_renamed(path, rel):
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+fix_strategies = [directory_renamed, \
+ mapfile_renamed, \
+ mapfile_and_directory_renamed]
+
+fmt = col_fmt = ""
+
+def set_terminal_output(dpdk_rel):
+ global fmt, col_fmt
+
+ fmt = '{:<50}'
+ col_fmt = fmt
+ for rel in dpdk_rel:
+ fmt += '{:<6}{:<6}'
+ col_fmt += '{:<12}'
+
+def set_csv_output(dpdk_rel):
+ global fmt, col_fmt
+
+ fmt = '{},'
+ col_fmt = fmt
+ for rel in dpdk_rel:
+ fmt += '{},{},'
+ col_fmt += '{},,'
+
+output_formats = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+directories = 'drivers, lib'
+
+def main():
+ global fmt, col_fmt, symbolMapGrammar
+
+ parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs')
+ parser.add_argument('--format-output', choices=['terminal','csv'], \
+ default='terminal')
+ parser.add_argument('--directory', choices=directories,
+ default=directories)
+ args = parser.parse_args()
+
+ dpdk_rel = get_dpdk_releases()
+
+ # set the output format
+ output_formats[args.format_output](dpdk_rel)
+
+ column_titles = ['mapfile'] + dpdk_rel
+ print(col_fmt.format(*column_titles))
+
+ symbolMapGrammar = symbolMapGrammar.format(get_abi_versions())
+ MAPParser = makeGrammar(symbolMapGrammar, {})
+
+ terminal_rows = get_terminal_rows()
+ row = 0
+
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ csym = [0] * 2
+ relsym = [str(path)]
+
+ for rel in dpdk_rel:
+ i = csym[0] = csym[1] = 0
+ abi_sections = None
+
+ tagfile = '{}:{}'.format(rel,path)
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE)
+
+ if result.stderr.startswith(b'fatal'):
+ result = None
+
+ while(result is None and i < len(fix_strategies)):
+ result = fix_strategies[i](path, rel)
+ i += 1
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = MAPParser(mapfile).abi()
+
+ if abi_sections is not None:
+ # which versions are present, and we care about
+ ignore = ['EXPERIMENTAL','INTERNAL']
+ found_ver = [ver \
+ for ver in abi_sections \
+ if ver not in ignore]
+
+ for ver in found_ver:
+ csym[0] += len(abi_sections[ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in abi_sections:
+ csym[1] = len(abi_sections['EXPERIMENTAL'])
+
+ relsym += csym
+
+ print(fmt.format(*relsym))
+ row += 1
+
+ if((terminal_rows>0) and ((row % terminal_rows) == 0)):
+ print(col_fmt.format(*column_titles))
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH] devtools: script to track map symbols
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
@ 2021-06-18 19:40 ` Stephen Hemminger
2021-06-21 9:18 ` Kinsella, Ray
2021-06-21 15:25 ` [dpdk-dev] [PATCH v3] " Ray Kinsella
` (11 subsequent siblings)
12 siblings, 1 reply; 52+ messages in thread
From: Stephen Hemminger @ 2021-06-18 19:40 UTC (permalink / raw)
To: Ray Kinsella; +Cc: dev, ferruh.yigit, thomas, ktraynor, bruce.richardson
On Fri, 18 Jun 2021 17:36:59 +0100
Ray Kinsella <mdr@ashroe.eu> wrote:
> Script to track growth of stable and experimental symbols
> over releases since v19.11.
>
> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
pylint reports some things that should be fixed. Don't worry about the naming style
and docstring but others should be addressed.
************* Module count_symbols
devtools/count_symbols.py:12:0: W0311: Bad indentation. Found 8 spaces, expected 4 (bad-indentation)
devtools/count_symbols.py:14:0: W0311: Bad indentation. Found 8 spaces, expected 4 (bad-indentation)
devtools/count_symbols.py:16:0: W0311: Bad indentation. Found 8 spaces, expected 4 (bad-indentation)
devtools/count_symbols.py:109:0: C0325: Unnecessary parens after 'if' keyword (superfluous-parens)
devtools/count_symbols.py:230:0: W0311: Bad indentation. Found 8 spaces, expected 4 (bad-indentation)
devtools/count_symbols.py:47:41: W1401: Anomalous backslash in string: '\.'. String constant might be missing an r prefix. (anomalous-backslash-in-string)
devtools/count_symbols.py:47:43: W1401: Anomalous backslash in string: '\d'. String constant might be missing an r prefix. (anomalous-backslash-in-string)
devtools/count_symbols.py:1:0: C0114: Missing module docstring (missing-module-docstring)
devtools/count_symbols.py:5:0: C0410: Multiple imports on one line (sys, os) (multiple-imports)
devtools/count_symbols.py:16:8: R1722: Consider using sys.exit() (consider-using-sys-exit)
devtools/count_symbols.py:18:0: C0103: Constant name "symbolMapGrammar" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:37:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:39:4: C0103: Variable name "s" doesn't conform to snake_case naming style (invalid-name)
devtools/count_symbols.py:40:4: C0103: Variable name "s" doesn't conform to snake_case naming style (invalid-name)
devtools/count_symbols.py:44:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:46:4: C0103: Variable name "s" doesn't conform to snake_case naming style (invalid-name)
devtools/count_symbols.py:50:13: W1510: Using subprocess.run without explicitly set `check` is not recommended. (subprocess-run-check)
devtools/count_symbols.py:66:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:70:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:78:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:82:13: W1510: Using subprocess.run without explicitly set `check` is not recommended. (subprocess-run-check)
devtools/count_symbols.py:91:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:94:13: W1510: Using subprocess.run without explicitly set `check` is not recommended. (subprocess-run-check)
devtools/count_symbols.py:112:17: W1510: Using subprocess.run without explicitly set `check` is not recommended. (subprocess-run-check)
devtools/count_symbols.py:124:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:133:0: C0103: Constant name "fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:133:6: C0103: Constant name "col_fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:135:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:136:4: W0603: Using the global statement (global-statement)
devtools/count_symbols.py:136:4: C0103: Constant name "fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:136:4: C0103: Constant name "col_fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:140:8: W0612: Unused variable 'rel' (unused-variable)
devtools/count_symbols.py:144:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:145:4: W0603: Using the global statement (global-statement)
devtools/count_symbols.py:145:4: C0103: Constant name "fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:145:4: C0103: Constant name "col_fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:149:8: W0612: Unused variable 'rel' (unused-variable)
devtools/count_symbols.py:156:0: C0103: Constant name "directories" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:158:0: C0116: Missing function or method docstring (missing-function-docstring)
devtools/count_symbols.py:158:0: R0914: Too many local variables (20/15) (too-many-locals)
devtools/count_symbols.py:159:4: W0603: Using the global statement (global-statement)
devtools/count_symbols.py:159:4: C0103: Constant name "fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:159:4: C0103: Constant name "col_fmt" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:159:4: C0103: Constant name "symbolMapGrammar" doesn't conform to UPPER_CASE naming style (invalid-name)
devtools/count_symbols.py:177:4: C0103: Variable name "MAPParser" doesn't conform to snake_case naming style (invalid-name)
devtools/count_symbols.py:192:25: W1510: Using subprocess.run without explicitly set `check` is not recommended. (subprocess-run-check)
devtools/count_symbols.py:5:0: W0611: Unused import sys (unused-import)
-----------------------------------
Your code has been rated at 6.27/10
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH] devtools: script to track map symbols
2021-06-18 19:40 ` Stephen Hemminger
@ 2021-06-21 9:18 ` Kinsella, Ray
2021-06-21 15:11 ` Ray Kinsella
0 siblings, 1 reply; 52+ messages in thread
From: Kinsella, Ray @ 2021-06-21 9:18 UTC (permalink / raw)
To: Stephen Hemminger; +Cc: dev, ferruh.yigit, thomas, ktraynor, bruce.richardson
>
> pylint reports some things that should be fixed. Don't worry about the naming style
> and docstring but others should be addressed.
[SNIP]
Ah, rookie mistake,
I ran checkpatch and thought that I was all good.
I will sort it out thanks.
Ray K
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH] devtools: script to track map symbols
2021-06-21 9:18 ` Kinsella, Ray
@ 2021-06-21 15:11 ` Ray Kinsella
0 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-06-21 15:11 UTC (permalink / raw)
To: dev; +Cc: stephen, ferruh.yigit, thomas, ktraynor, bruce.richardson, mdr
Script to track growth of stable and experimental symbols
over releases since v19.11.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/count_symbols.py | 230 ++++++++++++++++++++++++++++++++++++++
1 file changed, 230 insertions(+)
create mode 100755 devtools/count_symbols.py
diff --git a/devtools/count_symbols.py b/devtools/count_symbols.py
new file mode 100755
index 0000000000..7b29651044
--- /dev/null
+++ b/devtools/count_symbols.py
@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+from pathlib import Path
+import sys, os
+import subprocess
+import argparse
+import re
+import datetime
+
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ exit()
+
+symbolMapGrammar = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+"""
+
+#abi_ver = ['21', '20.0.1', '20.0', '20']
+
+def get_abi_versions():
+ year = datetime.date.today().year - 2000
+ s=" |".join(['\'{}\''.format(i) for i in reversed(range(21, year + 1)) ])
+ s = s + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return s
+
+def get_dpdk_releases():
+ year = datetime.date.today().year - 2000
+ s="|".join("{}".format(i) for i in range(19,year + 1))
+ pattern = re.compile('^\"v(' + s + ')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE)
+ if result.stderr.startswith(b'fatal'):
+ result = None
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [ tag.replace('\"','') \
+ for tag in reversed(tags) \
+ if pattern.match(tag) ][:-3]
+
+ return tags
+
+
+def get_terminal_rows():
+ rows, _ = os.popen('stty size', 'r').read().split()
+ return int(rows)
+
+def fix_directory_name(path):
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+# fix removal of the librte_ from the directory names
+def directory_renamed(path, rel):
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE)
+ if result.stderr.startswith(b'fatal'):
+ result = None
+
+ return result
+
+# fix renaming of map files
+def mapfile_renamed(path, rel):
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree', \
+ rel, str(path.parent) + '/'], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if(newfile is not None):
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE)
+ if result.stderr.startswith(b'fatal'):
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+# renaming of the map file & renaming of directory
+def mapfile_and_directory_renamed(path, rel):
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+fix_strategies = [directory_renamed, \
+ mapfile_renamed, \
+ mapfile_and_directory_renamed]
+
+fmt = col_fmt = ""
+
+def set_terminal_output(dpdk_rel):
+ global fmt, col_fmt
+
+ fmt = '{:<50}'
+ col_fmt = fmt
+ for rel in dpdk_rel:
+ fmt += '{:<6}{:<6}'
+ col_fmt += '{:<12}'
+
+def set_csv_output(dpdk_rel):
+ global fmt, col_fmt
+
+ fmt = '{},'
+ col_fmt = fmt
+ for rel in dpdk_rel:
+ fmt += '{},{},'
+ col_fmt += '{},,'
+
+output_formats = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+directories = 'drivers, lib'
+
+def main():
+ global fmt, col_fmt, symbolMapGrammar
+
+ parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs')
+ parser.add_argument('--format-output', choices=['terminal','csv'], \
+ default='terminal')
+ parser.add_argument('--directory', choices=directories,
+ default=directories)
+ args = parser.parse_args()
+
+ dpdk_rel = get_dpdk_releases()
+
+ # set the output format
+ output_formats[args.format_output](dpdk_rel)
+
+ column_titles = ['mapfile'] + dpdk_rel
+ print(col_fmt.format(*column_titles))
+
+ symbolMapGrammar = symbolMapGrammar.format(get_abi_versions())
+ MAPParser = makeGrammar(symbolMapGrammar, {})
+
+ terminal_rows = get_terminal_rows()
+ row = 0
+
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ csym = [0] * 2
+ relsym = [str(path)]
+
+ for rel in dpdk_rel:
+ i = csym[0] = csym[1] = 0
+ abi_sections = None
+
+ tagfile = '{}:{}'.format(rel,path)
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE)
+
+ if result.stderr.startswith(b'fatal'):
+ result = None
+
+ while(result is None and i < len(fix_strategies)):
+ result = fix_strategies[i](path, rel)
+ i += 1
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = MAPParser(mapfile).abi()
+
+ if abi_sections is not None:
+ # which versions are present, and we care about
+ ignore = ['EXPERIMENTAL','INTERNAL']
+ found_ver = [ver \
+ for ver in abi_sections \
+ if ver not in ignore]
+
+ for ver in found_ver:
+ csym[0] += len(abi_sections[ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in abi_sections:
+ csym[1] = len(abi_sections['EXPERIMENTAL'])
+
+ relsym += csym
+
+ print(fmt.format(*relsym))
+ row += 1
+
+ if((terminal_rows>0) and ((row % terminal_rows) == 0)):
+ print(col_fmt.format(*column_titles))
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v3] devtools: script to track map symbols
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
2021-06-18 19:40 ` Stephen Hemminger
@ 2021-06-21 15:25 ` Ray Kinsella
2021-06-21 15:35 ` [dpdk-dev] [PATCH v4] " Ray Kinsella
` (10 subsequent siblings)
12 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-06-21 15:25 UTC (permalink / raw)
To: dev; +Cc: stephen, ferruh.yigit, thomas, ktraynor, bruce.richardson, mdr
Script to track growth of stable and experimental symbols
over releases since v19.11.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
v2: reworked to fix pylint errors
v3: sent with the current in-reply-to
devtools/count_symbols.py | 262 ++++++++++++++++++++++++++++++++++++++
1 file changed, 262 insertions(+)
create mode 100755 devtools/count_symbols.py
diff --git a/devtools/count_symbols.py b/devtools/count_symbols.py
new file mode 100755
index 0000000000..30be09754f
--- /dev/null
+++ b/devtools/count_symbols.py
@@ -0,0 +1,262 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+'''Tool to count the number of symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+import re
+import datetime
+
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ sys.exit()
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+"""
+
+def get_abi_versions():
+ '''Returns a string of possible dpdk abi versions'''
+
+ year = datetime.date.today().year - 2000
+ tags = " |".join(['\'{}\''.format(i) \
+ for i in reversed(range(21, year + 1)) ])
+ tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return tags
+
+def get_dpdk_releases():
+ '''Returns a list of dpdk release tags names since v19.11'''
+
+ year = datetime.date.today().year - 2000
+ year_range = "|".join("{}".format(i) for i in range(19,year + 1))
+ pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ try:
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ print("Failed to interogate git for release tags")
+ sys.exit()
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [ tag.replace('\"','') \
+ for tag in reversed(tags) \
+ if pattern.match(tag) ][:-3]
+
+ return tags
+
+def fix_directory_name(path):
+ '''Prepend librte to the source directory name'''
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+def directory_renamed(path, rel):
+ '''Fix removal of the librte_ from the directory names'''
+
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ return result
+
+def mapfile_renamed(path, rel):
+ '''Fix renaming of map files'''
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree', \
+ rel, str(path.parent) + '/'], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if newfile is not None:
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+def mapfile_and_directory_renamed(path, rel):
+ '''Fix renaming of the map file & the source directory'''
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+def get_terminal_rows():
+ '''Find the number of rows in the terminal'''
+
+ rows, _ = os.popen('stty size', 'r').read().split()
+ return int(rows)
+
+class FormatOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + dpdk_releases
+
+ self.terminal_rows = get_terminal_rows()
+ self.row = 0
+
+ def set_terminal_output(self,dpdk_rel):
+ '''Set the output format to Tabbed Seperated Values'''
+
+ self.output_fmt = '{:<50}' + \
+ ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+ self.column_fmt = '{:50}' + \
+ ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+ def set_csv_output(self,dpdk_rel):
+ '''Set the output format to Comma Seperated Values'''
+
+ self.output_fmt = '{},' + \
+ ','.join(['{},{}'] * (len(dpdk_rel)))
+ self.column_fmt = '{},' + \
+ ','.join(['{},'] * (len(dpdk_rel)))
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+ self.row += 1
+
+ def print_row(self,symbols):
+ '''Print row of symbol values'''
+ print(self.output_fmt.format(*symbols))
+ self.row += 1
+
+ if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)):
+ self.print_columns()
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+SRC_DIRECTORIES = 'drivers, lib'
+IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL']
+FIX_STRATEGIES = [directory_renamed, \
+ mapfile_renamed, \
+ mapfile_and_directory_renamed]
+
+def count_release_symbols(map_parser, release, mapfile_path):
+ '''Count the symbols for a given release and mapfile'''
+ csym = [0] * 2
+ abi_sections = None
+
+ tagfile = '{}:{}'.format(release,mapfile_path)
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ for fix_strategy in FIX_STRATEGIES:
+ if result is not None:
+ break
+ result = fix_strategy(mapfile_path, release)
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = map_parser(mapfile).abi()
+
+ if abi_sections is not None:
+ # which versions are present, and we care about
+ found_ver = [ver \
+ for ver in abi_sections \
+ if ver not in IGNORE_SECTIONS]
+
+ for ver in found_ver:
+ csym[0] += len(abi_sections[ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in abi_sections:
+ csym[1] = len(abi_sections['EXPERIMENTAL'])
+
+ return csym
+
+def main():
+ '''Main entry point'''
+
+ parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs')
+ parser.add_argument('--format-output', choices=['terminal','csv'], \
+ default='terminal')
+ parser.add_argument('--directory', choices=SRC_DIRECTORIES,
+ default=SRC_DIRECTORIES)
+ args = parser.parse_args()
+
+ dpdk_releases = get_dpdk_releases()
+ format_output = FormatOutput(args.format_output, dpdk_releases)
+
+ map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+ map_parser = makeGrammar(map_grammar, {})
+
+ format_output.print_columns()
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ relsym = [str(path)]
+
+ for release in dpdk_releases:
+ csym = count_release_symbols(map_parser, release, path)
+ relsym += csym
+
+ format_output.print_row(relsym)
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v4] devtools: script to track map symbols
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
2021-06-18 19:40 ` Stephen Hemminger
2021-06-21 15:25 ` [dpdk-dev] [PATCH v3] " Ray Kinsella
@ 2021-06-21 15:35 ` Ray Kinsella
2021-06-21 18:14 ` Stephen Hemminger
2021-06-22 10:19 ` [dpdk-dev] [PATCH v5] " Ray Kinsella
` (9 subsequent siblings)
12 siblings, 1 reply; 52+ messages in thread
From: Ray Kinsella @ 2021-06-21 15:35 UTC (permalink / raw)
To: dev; +Cc: stephen, ferruh.yigit, thomas, ktraynor, bruce.richardson, mdr
Script to track growth of stable and experimental symbols
over releases since v19.11.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
v2: reworked to fix pylint errors
v3: sent with the correct in-reply-to
v4: fix typos picked up by the CI
devtools/count_symbols.py | 262 ++++++++++++++++++++++++++++++++++++++
1 file changed, 262 insertions(+)
create mode 100755 devtools/count_symbols.py
diff --git a/devtools/count_symbols.py b/devtools/count_symbols.py
new file mode 100755
index 0000000000..6194df0318
--- /dev/null
+++ b/devtools/count_symbols.py
@@ -0,0 +1,262 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+'''Tool to count the number of symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+import re
+import datetime
+
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ sys.exit()
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+"""
+
+def get_abi_versions():
+ '''Returns a string of possible dpdk abi versions'''
+
+ year = datetime.date.today().year - 2000
+ tags = " |".join(['\'{}\''.format(i) \
+ for i in reversed(range(21, year + 1)) ])
+ tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return tags
+
+def get_dpdk_releases():
+ '''Returns a list of dpdk release tags names since v19.11'''
+
+ year = datetime.date.today().year - 2000
+ year_range = "|".join("{}".format(i) for i in range(19,year + 1))
+ pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ try:
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ print("Failed to interogate git for release tags")
+ sys.exit()
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [ tag.replace('\"','') \
+ for tag in reversed(tags) \
+ if pattern.match(tag) ][:-3]
+
+ return tags
+
+def fix_directory_name(path):
+ '''Prepend librte to the source directory name'''
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+def directory_renamed(path, rel):
+ '''Fix removal of the librte_ from the directory names'''
+
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ return result
+
+def mapfile_renamed(path, rel):
+ '''Fix renaming of the map file'''
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree', \
+ rel, str(path.parent) + '/'], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if newfile is not None:
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+def mapfile_and_directory_renamed(path, rel):
+ '''Fix renaming of the map file & the source directory'''
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+def get_terminal_rows():
+ '''Find the number of rows in the terminal'''
+
+ rows, _ = os.popen('stty size', 'r').read().split()
+ return int(rows)
+
+class FormatOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + dpdk_releases
+
+ self.terminal_rows = get_terminal_rows()
+ self.row = 0
+
+ def set_terminal_output(self,dpdk_rel):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}' + \
+ ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+ self.column_fmt = '{:50}' + \
+ ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+ def set_csv_output(self,dpdk_rel):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},' + \
+ ','.join(['{},{}'] * (len(dpdk_rel)))
+ self.column_fmt = '{},' + \
+ ','.join(['{},'] * (len(dpdk_rel)))
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+ self.row += 1
+
+ def print_row(self,symbols):
+ '''Print row of symbol values'''
+ print(self.output_fmt.format(*symbols))
+ self.row += 1
+
+ if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)):
+ self.print_columns()
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+SRC_DIRECTORIES = 'drivers, lib'
+IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL']
+FIX_STRATEGIES = [directory_renamed, \
+ mapfile_renamed, \
+ mapfile_and_directory_renamed]
+
+def count_release_symbols(map_parser, release, mapfile_path):
+ '''Count the symbols for a given release and mapfile'''
+ csym = [0] * 2
+ abi_sections = None
+
+ tagfile = '{}:{}'.format(release,mapfile_path)
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ for fix_strategy in FIX_STRATEGIES:
+ if result is not None:
+ break
+ result = fix_strategy(mapfile_path, release)
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = map_parser(mapfile).abi()
+
+ if abi_sections is not None:
+ # which versions are present, and we care about
+ found_ver = [ver \
+ for ver in abi_sections \
+ if ver not in IGNORE_SECTIONS]
+
+ for ver in found_ver:
+ csym[0] += len(abi_sections[ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in abi_sections:
+ csym[1] = len(abi_sections['EXPERIMENTAL'])
+
+ return csym
+
+def main():
+ '''Main entry point'''
+
+ parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs')
+ parser.add_argument('--format-output', choices=['terminal','csv'], \
+ default='terminal')
+ parser.add_argument('--directory', choices=SRC_DIRECTORIES,
+ default=SRC_DIRECTORIES)
+ args = parser.parse_args()
+
+ dpdk_releases = get_dpdk_releases()
+ format_output = FormatOutput(args.format_output, dpdk_releases)
+
+ map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+ map_parser = makeGrammar(map_grammar, {})
+
+ format_output.print_columns()
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ relsym = [str(path)]
+
+ for release in dpdk_releases:
+ csym = count_release_symbols(map_parser, release, path)
+ relsym += csym
+
+ format_output.print_row(relsym)
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v4] devtools: script to track map symbols
2021-06-21 15:35 ` [dpdk-dev] [PATCH v4] " Ray Kinsella
@ 2021-06-21 18:14 ` Stephen Hemminger
0 siblings, 0 replies; 52+ messages in thread
From: Stephen Hemminger @ 2021-06-21 18:14 UTC (permalink / raw)
To: Ray Kinsella; +Cc: dev, ferruh.yigit, thomas, ktraynor, bruce.richardson
On Mon, 21 Jun 2021 16:35:31 +0100
Ray Kinsella <mdr@ashroe.eu> wrote:
> +def get_terminal_rows():
> + '''Find the number of rows in the terminal'''
> +
> + rows, _ = os.popen('stty size', 'r').read().split()
> + return int(rows)
> +
Use standard Python function os.get_terminal_size or shutil.get_terminal_size() instead.
That way it would be portable to other OS.
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v5] devtools: script to track map symbols
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
` (2 preceding siblings ...)
2021-06-21 15:35 ` [dpdk-dev] [PATCH v4] " Ray Kinsella
@ 2021-06-22 10:19 ` Ray Kinsella
2021-08-04 16:23 ` [dpdk-dev] [PATCH v6] " Ray Kinsella
` (8 subsequent siblings)
12 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-06-22 10:19 UTC (permalink / raw)
To: dev; +Cc: stephen, ferruh.yigit, thomas, ktraynor, bruce.richardson, mdr
Script to track growth of stable and experimental symbols
over releases since v19.11.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
v2: reworked to fix pylint errors
v3: sent with the correct in-reply-to
v4: fix typos picked up by the CI
v5: fix terminal_size & directory args
devtools/count_symbols.py | 262 ++++++++++++++++++++++++++++++++++++++
1 file changed, 262 insertions(+)
create mode 100755 devtools/count_symbols.py
diff --git a/devtools/count_symbols.py b/devtools/count_symbols.py
new file mode 100755
index 0000000000..96990f609f
--- /dev/null
+++ b/devtools/count_symbols.py
@@ -0,0 +1,262 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+'''Tool to count the number of symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+import re
+import datetime
+
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ sys.exit()
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+"""
+
+def get_abi_versions():
+ '''Returns a string of possible dpdk abi versions'''
+
+ year = datetime.date.today().year - 2000
+ tags = " |".join(['\'{}\''.format(i) \
+ for i in reversed(range(21, year + 1)) ])
+ tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return tags
+
+def get_dpdk_releases():
+ '''Returns a list of dpdk release tags names since v19.11'''
+
+ year = datetime.date.today().year - 2000
+ year_range = "|".join("{}".format(i) for i in range(19,year + 1))
+ pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ try:
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ print("Failed to interogate git for release tags")
+ sys.exit()
+
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [ tag.replace('\"','') \
+ for tag in reversed(tags) \
+ if pattern.match(tag) ][:-3]
+
+ return tags
+
+def fix_directory_name(path):
+ '''Prepend librte to the source directory name'''
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+def directory_renamed(path, rel):
+ '''Fix removal of the librte_ from the directory names'''
+
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ return result
+
+def mapfile_renamed(path, rel):
+ '''Fix renaming of the map file'''
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree', \
+ rel, str(path.parent) + '/'], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if newfile is not None:
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+def mapfile_and_directory_renamed(path, rel):
+ '''Fix renaming of the map file & the source directory'''
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+def get_terminal_rows():
+ '''Find the number of rows in the terminal'''
+
+ return os.get_terminal_size().lines
+
+class FormatOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + dpdk_releases
+
+ self.terminal_rows = get_terminal_rows()
+ self.row = 0
+
+ def set_terminal_output(self,dpdk_rel):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}' + \
+ ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+ self.column_fmt = '{:50}' + \
+ ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+ def set_csv_output(self,dpdk_rel):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},' + \
+ ','.join(['{},{}'] * (len(dpdk_rel)))
+ self.column_fmt = '{},' + \
+ ','.join(['{},'] * (len(dpdk_rel)))
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+ self.row += 1
+
+ def print_row(self,symbols):
+ '''Print row of symbol values'''
+ print(self.output_fmt.format(*symbols))
+ self.row += 1
+
+ if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)):
+ self.print_columns()
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+SRC_DIRECTORIES = 'drivers,lib'
+IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL']
+FIX_STRATEGIES = [directory_renamed, \
+ mapfile_renamed, \
+ mapfile_and_directory_renamed]
+
+def count_release_symbols(map_parser, release, mapfile_path):
+ '''Count the symbols for a given release and mapfile'''
+ csym = [0] * 2
+ abi_sections = None
+
+ tagfile = '{}:{}'.format(release,mapfile_path)
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ for fix_strategy in FIX_STRATEGIES:
+ if result is not None:
+ break
+ result = fix_strategy(mapfile_path, release)
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = map_parser(mapfile).abi()
+
+ if abi_sections is not None:
+ # which versions are present, and we care about
+ found_ver = [ver \
+ for ver in abi_sections \
+ if ver not in IGNORE_SECTIONS]
+
+ for ver in found_ver:
+ csym[0] += len(abi_sections[ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in abi_sections:
+ csym[1] = len(abi_sections['EXPERIMENTAL'])
+
+ return csym
+
+def main():
+ '''Main entry point'''
+
+ parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs')
+ parser.add_argument('--format-output', choices=['terminal','csv'], \
+ default='terminal')
+ parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','),
+ default=SRC_DIRECTORIES)
+ args = parser.parse_args()
+
+ dpdk_releases = get_dpdk_releases()
+ format_output = FormatOutput(args.format_output, dpdk_releases)
+
+ map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+ map_parser = makeGrammar(map_grammar, {})
+
+ format_output.print_columns()
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ relsym = [str(path)]
+
+ for release in dpdk_releases:
+ csym = count_release_symbols(map_parser, release, path)
+ relsym += csym
+
+ format_output.print_row(relsym)
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v6] devtools: script to track map symbols
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
` (3 preceding siblings ...)
2021-06-22 10:19 ` [dpdk-dev] [PATCH v5] " Ray Kinsella
@ 2021-08-04 16:23 ` Ray Kinsella
2021-08-04 16:27 ` [dpdk-dev] [PATCH v7] " Ray Kinsella
` (7 subsequent siblings)
12 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-08-04 16:23 UTC (permalink / raw)
To: dev; +Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr
This script tracks the growth of stable and experimental symbols
over releases since v19.11. The script has the ability to
count the added symbols between two dpdk releases, and to
list experimental symbols present in two dpdk releases
(expired symbols).
example usages:
Count symbols added since v19.11
$ devtools/symbol_tool.py count-symbols
Count symbols added since v20.11
$ devtools/symbol_tool.py count-symbols --releases v20.11,v21.05
List experimental symbols present in v20.11 and v21.05
$ devtools/symbol_tool.py list-expired --releases v20.11,v21.05
List experimental symbols in libraries only, present since v19.11
$ devtools/symbol_tool.py list-expired --directory lib
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
v2: reworked to fix pylint errors
v3: sent with the correct in-reply-to
v4: fix typos picked up by the CI
v5: fix terminal_size & directory args
v6: added list-expired, to list expired experimental symbols
devtools/symbol_tool.py | 377 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 377 insertions(+)
create mode 100755 devtools/symbol_tool.py
diff --git a/devtools/symbol_tool.py b/devtools/symbol_tool.py
new file mode 100755
index 0000000000..63969a131b
--- /dev/null
+++ b/devtools/symbol_tool.py
@@ -0,0 +1,377 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+'''Tool to count or list symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+import re
+import datetime
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ sys.exit()
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+"""
+
+def get_abi_versions():
+ '''Returns a string of possible dpdk abi versions'''
+
+ year = datetime.date.today().year - 2000
+ tags = " |".join(['\'{}\''.format(i) \
+ for i in reversed(range(21, year + 1)) ])
+ tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return tags
+
+def get_dpdk_releases():
+ '''Returns a list of dpdk release tags names since v19.11'''
+
+ year = datetime.date.today().year - 2000
+ year_range = "|".join("{}".format(i) for i in range(19,year + 1))
+ pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ try:
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ print("Failed to interogate git for release tags")
+ sys.exit()
+
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [ tag.replace('\"','') \
+ for tag in reversed(tags) \
+ if pattern.match(tag) ][:-3]
+
+ return tags
+
+def fix_directory_name(path):
+ '''Prepend librte to the source directory name'''
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+def directory_renamed(path, rel):
+ '''Fix removal of the librte_ from the directory names'''
+
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ return result
+
+def mapfile_renamed(path, rel):
+ '''Fix renaming of the map file'''
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree', \
+ rel, str(path.parent) + '/'], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if newfile is not None:
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+def mapfile_and_directory_renamed(path, rel):
+ '''Fix renaming of the map file & the source directory'''
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+FIX_STRATEGIES = [directory_renamed, \
+ mapfile_renamed, \
+ mapfile_and_directory_renamed]
+
+def get_symbols(map_parser, release, mapfile_path):
+ '''Count the symbols for a given release and mapfile'''
+ abi_sections = {}
+
+ tagfile = '{}:{}'.format(release,mapfile_path)
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ for fix_strategy in FIX_STRATEGIES:
+ if result is not None:
+ break
+ result = fix_strategy(mapfile_path, release)
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = map_parser(mapfile).abi()
+
+ return abi_sections
+
+def get_terminal_rows():
+ '''Find the number of rows in the terminal'''
+
+ try:
+ return os.get_terminal_size().lines
+ except IOError:
+ return 0
+
+class SymbolCountOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + dpdk_releases
+
+ self.terminal_rows = get_terminal_rows()
+ self.row = 0
+
+ def set_terminal_output(self,dpdk_rel):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}' + \
+ ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+ self.column_fmt = '{:50}' + \
+ ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+ def set_csv_output(self,dpdk_rel):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},' + \
+ ','.join(['{},{}'] * (len(dpdk_rel)))
+ self.column_fmt = '{},' + \
+ ','.join(['{},'] * (len(dpdk_rel)))
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+ self.row += 1
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+ print(self.output_fmt.format(*([mapfile] + symbols)))
+ self.row += 1
+
+ if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)):
+ self.print_columns()
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+class ListExpiredOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.terminal = True
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + \
+ ['expired (' + ','.join(dpdk_releases) + ')']
+
+ def set_terminal_output(self, _):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}{:<50}'
+ self.column_fmt = '{:50}{:50}'
+
+ def set_csv_output(self, _):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},{}'
+ self.column_fmt = '{},{}'
+ self.terminal = False
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+
+ for symbol in symbols:
+ print(self.output_fmt.format(mapfile,symbol))
+ if self.terminal :
+ mapfile = ''
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+class CountSymbolsAction:
+ ''' Logic to count symbols added since a give release '''
+ IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL']
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.symbols_count = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbol_count = experimental_count = 0
+
+ symbols = get_symbols(self.parser, release, self.path)
+
+ # which versions are present, and we care about
+ abi_vers = [abi_ver \
+ for abi_ver in symbols \
+ if abi_ver not in self.IGNORE_SECTIONS]
+
+ for abi_ver in abi_vers:
+ symbol_count += len(symbols[abi_ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental_count = len(symbols['EXPERIMENTAL'])
+
+ self.symbols_count += [symbol_count, experimental_count]
+
+ def __del__(self):
+ self.format_output.print_row(self.path.parent.name, self.symbols_count)
+
+class ListExpiredAction:
+ ''' Logic to list expired symbols between two releases '''
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.experimental_symbols = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbols = get_symbols(self.parser, release, self.path)
+ if 'EXPERIMENTAL' in symbols.keys():
+ self.experimental_symbols.append(symbols['EXPERIMENTAL'])
+
+ def __del__(self):
+ if len(self.experimental_symbols) != 2:
+ return
+
+ tmp = self.experimental_symbols
+ # find symbols present in both dpdk releases
+ intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]]
+
+ # check for empty set
+ if intersect_syms == []:
+ return
+
+ self.format_output.print_row(self.path.parent.name, intersect_syms)
+
+SRC_DIRECTORIES = 'drivers,lib'
+
+ACTIONS = {None: CountSymbolsAction, \
+ 'count-symbols': CountSymbolsAction, \
+ 'list-expired': ListExpiredAction}
+
+ACTION_OUTPUT = {None: SymbolCountOutput, \
+ 'count-symbols': SymbolCountOutput, \
+ 'list-expired': ListExpiredOutput}
+
+def main():
+ '''Main entry point'''
+
+ dpdk_releases = get_dpdk_releases()
+
+ parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs')
+ parser.add_argument('mode', choices=['count-symbols','list-expired'])
+ parser.add_argument('--format-output', choices=['terminal','csv'], \
+ default='terminal')
+ parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','),
+ default=SRC_DIRECTORIES)
+ parser.add_argument('--releases', \
+ help='2 x comma seperated release tags e.g. \'' \
+ + ','.join([dpdk_releases[0],dpdk_releases[-1]]) \
+ + '\'')
+ args = parser.parse_args()
+
+ if args.releases is not None:
+ dpdk_releases = args.releases.split(',')
+
+ if args.mode == 'list-expired':
+ if len(dpdk_releases) < 2:
+ sys.exit('Please specify two releases to compare ' \
+ 'in \'list-expired\' mode.')
+ dpdk_releases = [dpdk_releases[0], dpdk_releases[len(dpdk_releases) - 1]]
+
+ action = ACTIONS[args.mode]
+ format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases)
+
+ map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+ map_parser = makeGrammar(map_grammar, {})
+
+ format_output.print_columns()
+
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ release_action = action(path, map_parser, format_output)
+
+ for release in dpdk_releases:
+ release_action.add_mapfile(release)
+
+ # all the magic happens in the destructor
+ del release_action
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v7] devtools: script to track map symbols
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
` (4 preceding siblings ...)
2021-08-04 16:23 ` [dpdk-dev] [PATCH v6] " Ray Kinsella
@ 2021-08-04 16:27 ` Ray Kinsella
2021-08-06 17:54 ` [dpdk-dev] [PATCH v8 0/2] devtools: scripts to count and track symbols Ray Kinsella
` (6 subsequent siblings)
12 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-08-04 16:27 UTC (permalink / raw)
To: dev; +Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr
This script tracks the growth of stable and experimental symbols
over releases since v19.11. The script has the ability to
count the added symbols between two dpdk releases, and to
list experimental symbols present in two dpdk releases
(expired symbols).
example usages:
Count symbols added since v19.11
$ devtools/symbol_tool.py count-symbols
Count symbols added since v20.11
$ devtools/symbol_tool.py count-symbols --releases v20.11,v21.05
List experimental symbols present in v20.11 and v21.05
$ devtools/symbol_tool.py list-expired --releases v20.11,v21.05
List experimental symbols in libraries only, present since v19.11
$ devtools/symbol_tool.py list-expired --directory lib
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
v2: reworked to fix pylint errors
v3: sent with the correct in-reply-to
v4: fix typos picked up by the CI
v5: fix terminal_size & directory args
v6: added list-expired, to list expired experimental symbols
v7: fix typo in comments
devtools/symbol_tool.py | 377 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 377 insertions(+)
create mode 100755 devtools/symbol_tool.py
diff --git a/devtools/symbol_tool.py b/devtools/symbol_tool.py
new file mode 100755
index 0000000000..f2a2d43a15
--- /dev/null
+++ b/devtools/symbol_tool.py
@@ -0,0 +1,377 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+'''Tool to count or list symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+import re
+import datetime
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ sys.exit()
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+"""
+
+def get_abi_versions():
+ '''Returns a string of possible dpdk abi versions'''
+
+ year = datetime.date.today().year - 2000
+ tags = " |".join(['\'{}\''.format(i) \
+ for i in reversed(range(21, year + 1)) ])
+ tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return tags
+
+def get_dpdk_releases():
+ '''Returns a list of dpdk release tags names since v19.11'''
+
+ year = datetime.date.today().year - 2000
+ year_range = "|".join("{}".format(i) for i in range(19,year + 1))
+ pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ try:
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ print("Failed to interogate git for release tags")
+ sys.exit()
+
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [ tag.replace('\"','') \
+ for tag in reversed(tags) \
+ if pattern.match(tag) ][:-3]
+
+ return tags
+
+def fix_directory_name(path):
+ '''Prepend librte to the source directory name'''
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+def directory_renamed(path, rel):
+ '''Fix removal of the librte_ from the directory names'''
+
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ return result
+
+def mapfile_renamed(path, rel):
+ '''Fix renaming of the map file'''
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree', \
+ rel, str(path.parent) + '/'], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if newfile is not None:
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+def mapfile_and_directory_renamed(path, rel):
+ '''Fix renaming of the map file & the source directory'''
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+FIX_STRATEGIES = [directory_renamed, \
+ mapfile_renamed, \
+ mapfile_and_directory_renamed]
+
+def get_symbols(map_parser, release, mapfile_path):
+ '''Count the symbols for a given release and mapfile'''
+ abi_sections = {}
+
+ tagfile = '{}:{}'.format(release,mapfile_path)
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ for fix_strategy in FIX_STRATEGIES:
+ if result is not None:
+ break
+ result = fix_strategy(mapfile_path, release)
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = map_parser(mapfile).abi()
+
+ return abi_sections
+
+def get_terminal_rows():
+ '''Find the number of rows in the terminal'''
+
+ try:
+ return os.get_terminal_size().lines
+ except IOError:
+ return 0
+
+class SymbolCountOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + dpdk_releases
+
+ self.terminal_rows = get_terminal_rows()
+ self.row = 0
+
+ def set_terminal_output(self,dpdk_rel):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}' + \
+ ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+ self.column_fmt = '{:50}' + \
+ ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+ def set_csv_output(self,dpdk_rel):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},' + \
+ ','.join(['{},{}'] * (len(dpdk_rel)))
+ self.column_fmt = '{},' + \
+ ','.join(['{},'] * (len(dpdk_rel)))
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+ self.row += 1
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+ print(self.output_fmt.format(*([mapfile] + symbols)))
+ self.row += 1
+
+ if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)):
+ self.print_columns()
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+class ListExpiredOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.terminal = True
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + \
+ ['expired (' + ','.join(dpdk_releases) + ')']
+
+ def set_terminal_output(self, _):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}{:<50}'
+ self.column_fmt = '{:50}{:50}'
+
+ def set_csv_output(self, _):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},{}'
+ self.column_fmt = '{},{}'
+ self.terminal = False
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+
+ for symbol in symbols:
+ print(self.output_fmt.format(mapfile,symbol))
+ if self.terminal :
+ mapfile = ''
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+class CountSymbolsAction:
+ ''' Logic to count symbols added since a give release '''
+ IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL']
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.symbols_count = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbol_count = experimental_count = 0
+
+ symbols = get_symbols(self.parser, release, self.path)
+
+ # which versions are present, and we care about
+ abi_vers = [abi_ver \
+ for abi_ver in symbols \
+ if abi_ver not in self.IGNORE_SECTIONS]
+
+ for abi_ver in abi_vers:
+ symbol_count += len(symbols[abi_ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental_count = len(symbols['EXPERIMENTAL'])
+
+ self.symbols_count += [symbol_count, experimental_count]
+
+ def __del__(self):
+ self.format_output.print_row(self.path.parent.name, self.symbols_count)
+
+class ListExpiredAction:
+ ''' Logic to list expired symbols between two releases '''
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.experimental_symbols = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbols = get_symbols(self.parser, release, self.path)
+ if 'EXPERIMENTAL' in symbols.keys():
+ self.experimental_symbols.append(symbols['EXPERIMENTAL'])
+
+ def __del__(self):
+ if len(self.experimental_symbols) != 2:
+ return
+
+ tmp = self.experimental_symbols
+ # find symbols present in both dpdk releases
+ intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]]
+
+ # check for empty set
+ if intersect_syms == []:
+ return
+
+ self.format_output.print_row(self.path.parent.name, intersect_syms)
+
+SRC_DIRECTORIES = 'drivers,lib'
+
+ACTIONS = {None: CountSymbolsAction, \
+ 'count-symbols': CountSymbolsAction, \
+ 'list-expired': ListExpiredAction}
+
+ACTION_OUTPUT = {None: SymbolCountOutput, \
+ 'count-symbols': SymbolCountOutput, \
+ 'list-expired': ListExpiredOutput}
+
+def main():
+ '''Main entry point'''
+
+ dpdk_releases = get_dpdk_releases()
+
+ parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs')
+ parser.add_argument('mode', choices=['count-symbols','list-expired'])
+ parser.add_argument('--format-output', choices=['terminal','csv'], \
+ default='terminal')
+ parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','),
+ default=SRC_DIRECTORIES)
+ parser.add_argument('--releases', \
+ help='2 x comma separated release tags e.g. \'' \
+ + ','.join([dpdk_releases[0],dpdk_releases[-1]]) \
+ + '\'')
+ args = parser.parse_args()
+
+ if args.releases is not None:
+ dpdk_releases = args.releases.split(',')
+
+ if args.mode == 'list-expired':
+ if len(dpdk_releases) < 2:
+ sys.exit('Please specify two releases to compare ' \
+ 'in \'list-expired\' mode.')
+ dpdk_releases = [dpdk_releases[0], dpdk_releases[len(dpdk_releases) - 1]]
+
+ action = ACTIONS[args.mode]
+ format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases)
+
+ map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+ map_parser = makeGrammar(map_grammar, {})
+
+ format_output.print_columns()
+
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ release_action = action(path, map_parser, format_output)
+
+ for release in dpdk_releases:
+ release_action.add_mapfile(release)
+
+ # all the magic happens in the destructor
+ del release_action
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v8 0/2] devtools: scripts to count and track symbols
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
` (5 preceding siblings ...)
2021-08-04 16:27 ` [dpdk-dev] [PATCH v7] " Ray Kinsella
@ 2021-08-06 17:54 ` Ray Kinsella
2021-08-06 17:54 ` [dpdk-dev] [PATCH v8 1/2] devtools: script to track map symbols Ray Kinsella
2021-08-06 17:54 ` [dpdk-dev] [PATCH v8 2/2] devtools: script to send notifications of expired symbols Ray Kinsella
2021-08-09 12:53 ` [dpdk-dev] [PATCH v9 0/2] devtools: scripts to count and track symbols Ray Kinsella
` (5 subsequent siblings)
12 siblings, 2 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-08-06 17:54 UTC (permalink / raw)
To: dev; +Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr
Scripts to count and track the lifecycle of DPDK symbols.
The symbol-tool script reports on the growth of symbols over releases
and list expired symbols. The notify-symbol-maintainers script
consumes the input from symbol-tool and generates email notifications
of expired symbols.
v2: reworked to fix pylint errors
v3: sent with the correct in-reply-to
v4: fix typos picked up by the CI
v5: fix terminal_size & directory args
v6: added list-expired, to list expired experimental symbols
v7: fix typo in comments
v8: added tool to notify maintainers of expired symbols
Ray Kinsella (2):
devtools: script to track map symbols
devtools: script to send notifications of expired symbols
devtools/notify-symbol-maintainers.py | 224 ++++++++++++++
devtools/symbol-tool.py | 402 ++++++++++++++++++++++++++
2 files changed, 626 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
create mode 100755 devtools/symbol-tool.py
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v8 1/2] devtools: script to track map symbols
2021-08-06 17:54 ` [dpdk-dev] [PATCH v8 0/2] devtools: scripts to count and track symbols Ray Kinsella
@ 2021-08-06 17:54 ` Ray Kinsella
2021-08-06 17:54 ` [dpdk-dev] [PATCH v8 2/2] devtools: script to send notifications of expired symbols Ray Kinsella
1 sibling, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-08-06 17:54 UTC (permalink / raw)
To: dev; +Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr
This script tracks the growth of stable and experimental symbols
over releases since v19.11. The script has the ability to
count the added symbols between two dpdk releases, and to
list experimental symbols present in two dpdk releases
(expired symbols).
example usages:
Count symbols added since v19.11
$ devtools/symbol-tool.py count-symbols
Count symbols added since v20.11
$ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05
List experimental symbols present in v20.11 and v21.05
$ devtools/symbol-tool.py list-expired --releases v20.11,v21.05
List experimental symbols in libraries only, present since v19.11
$ devtools/symbol-tool.py list-expired --directory lib
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/symbol-tool.py | 402 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 402 insertions(+)
create mode 100755 devtools/symbol-tool.py
diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py
new file mode 100755
index 0000000000..39727c9a32
--- /dev/null
+++ b/devtools/symbol-tool.py
@@ -0,0 +1,402 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+'''Tool to count or list symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import re
+import datetime
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ sys.exit()
+
+DESCRIPTION = '''
+This script tracks the growth of stable and experimental symbols
+over releases since v19.11. The script has the ability to
+count the added symbols between two dpdk releases, and to
+list experimental symbols present in two dpdk releases
+(expired symbols).
+
+example usages:
+
+Count symbols added since v19.11
+$ devtools/symbol-tool.py count-symbols
+
+Count symbols added since v20.11
+$ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05
+
+List experimental symbols present in v20.11 and v21.05
+$ devtools/symbol-tool.py list-expired --releases v20.11,v21.05
+
+List experimental symbols in libraries only, present since v19.11
+$ devtools/symbol-tool.py list-expired --directory lib
+'''
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+"""
+
+def get_abi_versions():
+ '''Returns a string of possible dpdk abi versions'''
+
+ year = datetime.date.today().year - 2000
+ tags = " |".join(['\'{}\''.format(i) \
+ for i in reversed(range(21, year + 1)) ])
+ tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return tags
+
+def get_dpdk_releases():
+ '''Returns a list of dpdk release tags names since v19.11'''
+
+ year = datetime.date.today().year - 2000
+ year_range = "|".join("{}".format(i) for i in range(19,year + 1))
+ pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ try:
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ print("Failed to interogate git for release tags")
+ sys.exit()
+
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [ tag.replace('\"','') \
+ for tag in reversed(tags) \
+ if pattern.match(tag) ][:-3]
+
+ return tags
+
+def fix_directory_name(path):
+ '''Prepend librte to the source directory name'''
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+def directory_renamed(path, rel):
+ '''Fix removal of the librte_ from the directory names'''
+
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ return result
+
+def mapfile_renamed(path, rel):
+ '''Fix renaming of the map file'''
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree', \
+ rel, str(path.parent) + '/'], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if newfile is not None:
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+def mapfile_and_directory_renamed(path, rel):
+ '''Fix renaming of the map file & the source directory'''
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+FIX_STRATEGIES = [directory_renamed, \
+ mapfile_renamed, \
+ mapfile_and_directory_renamed]
+
+def get_symbols(map_parser, release, mapfile_path):
+ '''Count the symbols for a given release and mapfile'''
+ abi_sections = {}
+
+ tagfile = '{}:{}'.format(release,mapfile_path)
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ for fix_strategy in FIX_STRATEGIES:
+ if result is not None:
+ break
+ result = fix_strategy(mapfile_path, release)
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = map_parser(mapfile).abi()
+
+ return abi_sections
+
+def get_terminal_rows():
+ '''Find the number of rows in the terminal'''
+
+ try:
+ return os.get_terminal_size().lines
+ except IOError:
+ return 0
+
+class SymbolCountOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + dpdk_releases
+
+ self.terminal_rows = get_terminal_rows()
+ self.row = 0
+
+ def set_terminal_output(self,dpdk_rel):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}' + \
+ ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+ self.column_fmt = '{:50}' + \
+ ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+ def set_csv_output(self,dpdk_rel):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},' + \
+ ','.join(['{},{}'] * (len(dpdk_rel)))
+ self.column_fmt = '{},' + \
+ ','.join(['{},'] * (len(dpdk_rel)))
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+ self.row += 1
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+ print(self.output_fmt.format(*([mapfile] + symbols)))
+ self.row += 1
+
+ if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)):
+ self.print_columns()
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+class ListExpiredOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.terminal = True
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + \
+ ['expired (' + ','.join(dpdk_releases) + ')']
+
+ def set_terminal_output(self, _):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}{:<50}'
+ self.column_fmt = '{:50}{:50}'
+
+ def set_csv_output(self, _):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},{}'
+ self.column_fmt = '{},{}'
+ self.terminal = False
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+
+ for symbol in symbols:
+ print(self.output_fmt.format(mapfile,symbol))
+ if self.terminal :
+ mapfile = ''
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+class CountSymbolsAction:
+ ''' Logic to count symbols added since a give release '''
+ IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL']
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.symbols_count = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbol_count = experimental_count = 0
+
+ symbols = get_symbols(self.parser, release, self.path)
+
+ # which versions are present, and we care about
+ abi_vers = [abi_ver \
+ for abi_ver in symbols \
+ if abi_ver not in self.IGNORE_SECTIONS]
+
+ for abi_ver in abi_vers:
+ symbol_count += len(symbols[abi_ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental_count = len(symbols['EXPERIMENTAL'])
+
+ self.symbols_count += [symbol_count, experimental_count]
+
+ def __del__(self):
+ self.format_output.print_row(self.path.parent, self.symbols_count)
+
+class ListExpiredAction:
+ ''' Logic to list expired symbols between two releases '''
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.experimental_symbols = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbols = get_symbols(self.parser, release, self.path)
+ if 'EXPERIMENTAL' in symbols.keys():
+ self.experimental_symbols.append(symbols['EXPERIMENTAL'])
+
+ def __del__(self):
+ if len(self.experimental_symbols) != 2:
+ return
+
+ tmp = self.experimental_symbols
+ # find symbols present in both dpdk releases
+ intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]]
+
+ # check for empty set
+ if intersect_syms == []:
+ return
+
+ self.format_output.print_row(self.path.parent, intersect_syms)
+
+SRC_DIRECTORIES = 'drivers,lib'
+
+ACTIONS = {None: CountSymbolsAction, \
+ 'count-symbols': CountSymbolsAction, \
+ 'list-expired': ListExpiredAction}
+
+ACTION_OUTPUT = {None: SymbolCountOutput, \
+ 'count-symbols': SymbolCountOutput, \
+ 'list-expired': ListExpiredOutput}
+
+def main():
+ '''Main entry point'''
+
+ dpdk_releases = get_dpdk_releases()
+
+ parser = argparse.ArgumentParser(description=DESCRIPTION, \
+ formatter_class=RawTextHelpFormatter
+ )
+ parser.add_argument('mode', choices=['count-symbols','list-expired'])
+ parser.add_argument('--format-output', choices=['terminal','csv'], \
+ default='terminal')
+ parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','),
+ default=SRC_DIRECTORIES)
+ parser.add_argument('--releases', \
+ help='2 x comma separated release tags e.g. \'' \
+ + ','.join([dpdk_releases[0],dpdk_releases[-1]]) \
+ + '\'')
+ args = parser.parse_args()
+
+ if args.releases is not None:
+ dpdk_releases = args.releases.split(',')
+
+ if args.mode == 'list-expired':
+ if len(dpdk_releases) < 2:
+ sys.exit('Please specify two releases to compare ' \
+ 'in \'list-expired\' mode.')
+ dpdk_releases = [dpdk_releases[0], dpdk_releases[len(dpdk_releases) - 1]]
+
+ action = ACTIONS[args.mode]
+ format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases)
+
+ map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+ map_parser = makeGrammar(map_grammar, {})
+
+ format_output.print_columns()
+
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ release_action = action(path, map_parser, format_output)
+
+ for release in dpdk_releases:
+ release_action.add_mapfile(release)
+
+ # all the magic happens in the destructor
+ del release_action
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v8 2/2] devtools: script to send notifications of expired symbols
2021-08-06 17:54 ` [dpdk-dev] [PATCH v8 0/2] devtools: scripts to count and track symbols Ray Kinsella
2021-08-06 17:54 ` [dpdk-dev] [PATCH v8 1/2] devtools: script to track map symbols Ray Kinsella
@ 2021-08-06 17:54 ` Ray Kinsella
1 sibling, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-08-06 17:54 UTC (permalink / raw)
To: dev; +Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr
Use this script with the output of the DPDK symbol tool, to notify
maintainers of expired symbols by email. You need to define the environment
variable DPDK_GETMAINTAINER_PATH, for this tool to work.
Use terminal output to review the emails before sending.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
devtools/notify_expired_symbols.py --format-output terminal
Then use email output to send the emails to the maintainers.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
--smtp-server <server> --sender <someone@somewhere.com> \
--password <password>
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/notify-symbol-maintainers.py | 224 ++++++++++++++++++++++++++
1 file changed, 224 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py
new file mode 100755
index 0000000000..447f88bb03
--- /dev/null
+++ b/devtools/notify-symbol-maintainers.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+'''Tool to notify maintainers of expired symbols'''
+import smtplib
+import ssl
+import sys
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import time
+from email.message import EmailMessage
+
+DESCRIPTION = '''
+Use this script with the output of the DPDK symbol tool, to notify maintainers
+of expired symbols by email. You need to define the environment variable
+DPDK_GETMAINTAINER_PATH, for this tool to work.
+
+Use terminal output to review the emails before sending.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+devtools/notify_expired_symbols.py --format-output terminal
+
+Then use email output to send the emails to the maintainers.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+--smtp-server <server> --sender <someone@somewhere.com> --password <password>
+'''
+
+EMAIL_TEMPLATE = '''Hi there,
+
+Please note the symbols listed below have expired. In line with the DPDK ABI
+policy, they should be scheduled for removal, in the next DPDK release.
+
+For more information, please see the DPDK ABI Policy, section 3.5.3.
+https://doc.dpdk.org/guides/contributing/abi_policy.html
+
+Thanks,
+
+The DPDK Symbol Bot
+
+'''
+
+default_maintainers = ['Ray Kinsella <mdr@ashroe.eu>', \
+ 'Thomas Monjalon <thomas@monjalon.net>']
+get_maintainer = ['devtools/get-maintainer.sh', \
+ '--email', '-f']
+
+def get_maintainers(libpath):
+ '''Get the maintainers for given library'''
+ try:
+ cmd = get_maintainer + [libpath]
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ if result is not None:
+ email = result.stdout.decode('utf-8')
+ if email == '':
+ email = default_maintainers
+ else:
+ email = list(filter(None,email.split('\n')))
+ else:
+ email = default_maintainers
+
+ return email
+
+def get_message(library, symbols):
+ '''Build email message from symbols, config and maintainers'''
+ message = {}
+ maintainers = get_maintainers(library)
+
+ message['To'] = maintainers
+ if maintainers != default_maintainers:
+ message['CC'] = default_maintainers
+
+ message['Subject'] = 'Expired symbols in {}\n'.format(library)
+
+ body = EMAIL_TEMPLATE
+ for sym in symbols:
+ body += ('{}\n'.format(sym))
+
+ message['Body'] = body
+
+ return message
+
+class OutputEmail():
+ '''Format the output for email'''
+ def __init__(self, config):
+ self.config = config
+
+ self.terminal = OutputTerminal(config)
+ context = ssl.create_default_context()
+
+ # Try to log in to server and send email
+ try:
+ self.server = smtplib.SMTP(config['smtp_server'], 587)
+ self.server.starttls(context=context) # Secure the connection
+ self.server.login(config['sender'], config['password'])
+ except Exception as exception:
+ print(exception)
+ raise exception
+
+ def message(self,message):
+ '''send email'''
+ self.terminal.message(message)
+
+ msg = EmailMessage()
+ msg.set_content(message.pop('Body'))
+
+ for key in message.keys():
+ msg[key] = message[key]
+
+ msg['From'] = self.config['sender']
+ msg['Reply-To'] = 'no-reply@dpdk.org'
+
+ self.server.send_message(msg)
+
+ time.sleep(1)
+
+ def __del__(self):
+ self.server.quit()
+
+class OutputTerminal(): # pylint: disable=too-few-public-methods
+ '''Format the output for the terminal'''
+ def __init__(self, config):
+ self.config = config
+
+ def message(self,message):
+ '''Print email to terminal'''
+ terminal = 'To:' + ', '.join(message['To']) + '\n'
+ if 'sender' in self.config.keys():
+ terminal += 'From:' + self.config['sender'] + '\n'
+
+ terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n'
+ if 'CC' in message.keys():
+ terminal += 'CC:' + ', '.join(message['CC']) + '\n'
+
+ terminal += 'Subject:' + message['Subject'] + '\n'
+ terminal += 'Body:' + message['Body'] + '\n'
+
+ print(terminal)
+ print('-' * 80)
+
+def parse_config(args):
+ '''put the command line args in the right places'''
+ config = {}
+ error_msg = None
+
+ outputs = {
+ None : OutputTerminal,
+ 'terminal' : OutputTerminal,
+ 'email' : OutputEmail
+ }
+
+ if args.format_output == 'email':
+ if args.smtp_server is None:
+ error_msg = 'SMTP server'
+ else:
+ config['smtp_server'] = args.smtp_server
+
+ if args.sender is None:
+ error_msg = 'sender'
+ else:
+ config['sender'] = args.sender
+
+ if args.password is None:
+ error_msg = 'password'
+ else:
+ config['password'] = args.password
+
+ if error_msg is not None:
+ print('Please specify a {} for email output'.format(error_msg))
+ return None
+
+ config['output'] = outputs[args.format_output]
+ return config
+
+def main():
+ '''Main entry point'''
+ parser = argparse.ArgumentParser(description=DESCRIPTION, \
+ formatter_class=RawTextHelpFormatter)
+ parser.add_argument('--format-output', choices=['terminal','email'], \
+ default='terminal')
+ parser.add_argument('--smtp-server')
+ parser.add_argument('--password')
+ parser.add_argument('--sender')
+
+ args = parser.parse_args()
+ config = parse_config(args)
+ if config is None:
+ return
+
+ symbols = []
+ lastlib = library = ''
+
+ output = config['output'](config)
+
+ for line in sys.stdin:
+ line = line.rstrip('\n')
+ library, symbol = [line[:line.find(',')], \
+ line[line.find(',') + 1: len(line)]]
+ if library == 'mapfile':
+ continue
+
+ if library != lastlib:
+ message = get_message(lastlib, symbols)
+ output.message(message)
+ symbols = []
+
+ lastlib = library
+ symbols = symbols + [symbol]
+
+ #print the last library
+ message = get_message(lastlib, symbols)
+ output.message(message)
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v9 0/2] devtools: scripts to count and track symbols
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
` (6 preceding siblings ...)
2021-08-06 17:54 ` [dpdk-dev] [PATCH v8 0/2] devtools: scripts to count and track symbols Ray Kinsella
@ 2021-08-09 12:53 ` Ray Kinsella
2021-08-09 12:53 ` [dpdk-dev] [PATCH v9 1/2] devtools: script to track symbols over releases Ray Kinsella
2021-08-09 12:53 ` [dpdk-dev] [PATCH v9 2/2] devtools: script to send notifications of expired symbols Ray Kinsella
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols Ray Kinsella
` (4 subsequent siblings)
12 siblings, 2 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-08-09 12:53 UTC (permalink / raw)
To: dev; +Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr
Scripts to count and track the lifecycle of DPDK symbols.
The symbol-tool script reports on the growth of symbols over releases
and list expired symbols. The notify-symbol-maintainers script
consumes the input from symbol-tool and generates email notifications
of expired symbols.
v2: reworked to fix pylint errors
v3: sent with the correct in-reply-to
v4: fix typos picked up by the CI
v5: fix terminal_size & directory args
v6: added list-expired, to list expired experimental symbols
v7: fix typo in comments
v8: added tool to notify maintainers of expired symbols
v9: removed hardcoded emails addressed and script names
Ray Kinsella (2):
devtools: script to track symbols over releases
devtools: script to send notifications of expired symbols
devtools/notify-symbol-maintainers.py | 234 +++++++++++++++
devtools/symbol-tool.py | 402 ++++++++++++++++++++++++++
2 files changed, 636 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
create mode 100755 devtools/symbol-tool.py
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v9 1/2] devtools: script to track symbols over releases
2021-08-09 12:53 ` [dpdk-dev] [PATCH v9 0/2] devtools: scripts to count and track symbols Ray Kinsella
@ 2021-08-09 12:53 ` Ray Kinsella
2021-08-09 12:53 ` [dpdk-dev] [PATCH v9 2/2] devtools: script to send notifications of expired symbols Ray Kinsella
1 sibling, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-08-09 12:53 UTC (permalink / raw)
To: dev; +Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr
This script tracks the growth of stable and experimental symbols
over releases since v19.11. The script has the ability to
count the added symbols between two dpdk releases, and to
list experimental symbols present in two dpdk releases
(expired symbols).
example usages:
Count symbols added since v19.11
$ devtools/symbol-tool.py count-symbols
Count symbols added since v20.11
$ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05
List experimental symbols present in v20.11 and v21.05
$ devtools/symbol-tool.py list-expired --releases v20.11,v21.05
List experimental symbols in libraries only, present since v19.11
$ devtools/symbol-tool.py list-expired --directory lib
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/symbol-tool.py | 402 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 402 insertions(+)
create mode 100755 devtools/symbol-tool.py
diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py
new file mode 100755
index 0000000000..4a357579dc
--- /dev/null
+++ b/devtools/symbol-tool.py
@@ -0,0 +1,402 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+'''Tool to count or list symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import re
+import datetime
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ sys.exit()
+
+DESCRIPTION = '''
+This script tracks the growth of stable and experimental symbols
+over releases since v19.11. The script has the ability to
+count the added symbols between two dpdk releases, and to
+list experimental symbols present in two dpdk releases
+(expired symbols).
+
+example usages:
+
+Count symbols added since v19.11
+$ {s} count-symbols
+
+Count symbols added since v20.11
+$ {s} count-symbols --releases v20.11,v21.05
+
+List experimental symbols present in v20.11 and v21.05
+$ {s} list-expired --releases v20.11,v21.05
+
+List experimental symbols in libraries only, present since v19.11
+$ {s} list-expired --directory lib
+'''
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+"""
+
+def get_abi_versions():
+ '''Returns a string of possible dpdk abi versions'''
+
+ year = datetime.date.today().year - 2000
+ tags = " |".join(['\'{}\''.format(i) \
+ for i in reversed(range(21, year + 1)) ])
+ tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return tags
+
+def get_dpdk_releases():
+ '''Returns a list of dpdk release tags names since v19.11'''
+
+ year = datetime.date.today().year - 2000
+ year_range = "|".join("{}".format(i) for i in range(19,year + 1))
+ pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ try:
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ print("Failed to interogate git for release tags")
+ sys.exit()
+
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [ tag.replace('\"','') \
+ for tag in reversed(tags) \
+ if pattern.match(tag) ][:-3]
+
+ return tags
+
+def fix_directory_name(path):
+ '''Prepend librte to the source directory name'''
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+def directory_renamed(path, rel):
+ '''Fix removal of the librte_ from the directory names'''
+
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ return result
+
+def mapfile_renamed(path, rel):
+ '''Fix renaming of the map file'''
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree', \
+ rel, str(path.parent) + '/'], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if newfile is not None:
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+def mapfile_and_directory_renamed(path, rel):
+ '''Fix renaming of the map file & the source directory'''
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+FIX_STRATEGIES = [directory_renamed, \
+ mapfile_renamed, \
+ mapfile_and_directory_renamed]
+
+def get_symbols(map_parser, release, mapfile_path):
+ '''Count the symbols for a given release and mapfile'''
+ abi_sections = {}
+
+ tagfile = '{}:{}'.format(release,mapfile_path)
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ for fix_strategy in FIX_STRATEGIES:
+ if result is not None:
+ break
+ result = fix_strategy(mapfile_path, release)
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = map_parser(mapfile).abi()
+
+ return abi_sections
+
+def get_terminal_rows():
+ '''Find the number of rows in the terminal'''
+
+ try:
+ return os.get_terminal_size().lines
+ except IOError:
+ return 0
+
+class SymbolCountOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + dpdk_releases
+
+ self.terminal_rows = get_terminal_rows()
+ self.row = 0
+
+ def set_terminal_output(self,dpdk_rel):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}' + \
+ ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+ self.column_fmt = '{:50}' + \
+ ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+ def set_csv_output(self,dpdk_rel):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},' + \
+ ','.join(['{},{}'] * (len(dpdk_rel)))
+ self.column_fmt = '{},' + \
+ ','.join(['{},'] * (len(dpdk_rel)))
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+ self.row += 1
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+ print(self.output_fmt.format(*([mapfile] + symbols)))
+ self.row += 1
+
+ if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)):
+ self.print_columns()
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+class ListExpiredOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.terminal = True
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + \
+ ['expired (' + ','.join(dpdk_releases) + ')']
+
+ def set_terminal_output(self, _):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}{:<50}'
+ self.column_fmt = '{:50}{:50}'
+
+ def set_csv_output(self, _):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},{}'
+ self.column_fmt = '{},{}'
+ self.terminal = False
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+
+ for symbol in symbols:
+ print(self.output_fmt.format(mapfile,symbol))
+ if self.terminal :
+ mapfile = ''
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+class CountSymbolsAction:
+ ''' Logic to count symbols added since a give release '''
+ IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL']
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.symbols_count = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbol_count = experimental_count = 0
+
+ symbols = get_symbols(self.parser, release, self.path)
+
+ # which versions are present, and we care about
+ abi_vers = [abi_ver \
+ for abi_ver in symbols \
+ if abi_ver not in self.IGNORE_SECTIONS]
+
+ for abi_ver in abi_vers:
+ symbol_count += len(symbols[abi_ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental_count = len(symbols['EXPERIMENTAL'])
+
+ self.symbols_count += [symbol_count, experimental_count]
+
+ def __del__(self):
+ self.format_output.print_row(self.path.parent, self.symbols_count)
+
+class ListExpiredAction:
+ ''' Logic to list expired symbols between two releases '''
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.experimental_symbols = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbols = get_symbols(self.parser, release, self.path)
+ if 'EXPERIMENTAL' in symbols.keys():
+ self.experimental_symbols.append(symbols['EXPERIMENTAL'])
+
+ def __del__(self):
+ if len(self.experimental_symbols) != 2:
+ return
+
+ tmp = self.experimental_symbols
+ # find symbols present in both dpdk releases
+ intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]]
+
+ # check for empty set
+ if intersect_syms == []:
+ return
+
+ self.format_output.print_row(self.path.parent, intersect_syms)
+
+SRC_DIRECTORIES = 'drivers,lib'
+
+ACTIONS = {None: CountSymbolsAction, \
+ 'count-symbols': CountSymbolsAction, \
+ 'list-expired': ListExpiredAction}
+
+ACTION_OUTPUT = {None: SymbolCountOutput, \
+ 'count-symbols': SymbolCountOutput, \
+ 'list-expired': ListExpiredOutput}
+
+def main():
+ '''Main entry point'''
+
+ dpdk_releases = get_dpdk_releases()
+
+ parser = argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), \
+ formatter_class=RawTextHelpFormatter
+ )
+ parser.add_argument('mode', choices=['count-symbols','list-expired'])
+ parser.add_argument('--format-output', choices=['terminal','csv'], \
+ default='terminal')
+ parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','),
+ default=SRC_DIRECTORIES)
+ parser.add_argument('--releases', \
+ help='2 x comma separated release tags e.g. \'' \
+ + ','.join([dpdk_releases[0],dpdk_releases[-1]]) \
+ + '\'')
+ args = parser.parse_args()
+
+ if args.releases is not None:
+ dpdk_releases = args.releases.split(',')
+
+ if args.mode == 'list-expired':
+ if len(dpdk_releases) < 2:
+ sys.exit('Please specify two releases to compare ' \
+ 'in \'list-expired\' mode.')
+ dpdk_releases = [dpdk_releases[0], dpdk_releases[len(dpdk_releases) - 1]]
+
+ action = ACTIONS[args.mode]
+ format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases)
+
+ map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+ map_parser = makeGrammar(map_grammar, {})
+
+ format_output.print_columns()
+
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ release_action = action(path, map_parser, format_output)
+
+ for release in dpdk_releases:
+ release_action.add_mapfile(release)
+
+ # all the magic happens in the destructor
+ del release_action
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v9 2/2] devtools: script to send notifications of expired symbols
2021-08-09 12:53 ` [dpdk-dev] [PATCH v9 0/2] devtools: scripts to count and track symbols Ray Kinsella
2021-08-09 12:53 ` [dpdk-dev] [PATCH v9 1/2] devtools: script to track symbols over releases Ray Kinsella
@ 2021-08-09 12:53 ` Ray Kinsella
1 sibling, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-08-09 12:53 UTC (permalink / raw)
To: dev; +Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr
Use this script with the output of the DPDK symbol tool, to notify
maintainers of expired symbols by email. You need to define the environment
variable DPDK_GETMAINTAINER_PATH for this tool to work.
Use terminal output to review the emails before sending.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
devtools/notify_expired_symbols.py --format-output terminal
Then use email output to send the emails to the maintainers.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
devtools/notify_expired_symbols.py --format-output email \
--smtp-server <server> --sender <someone@somewhere.com> \
--password <password>
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/notify-symbol-maintainers.py | 234 ++++++++++++++++++++++++++
1 file changed, 234 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py
new file mode 100755
index 0000000000..a6c27b067c
--- /dev/null
+++ b/devtools/notify-symbol-maintainers.py
@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+'''Tool to notify maintainers of expired symbols'''
+import smtplib
+import ssl
+import sys
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import time
+from email.message import EmailMessage
+
+DESCRIPTION = '''
+Use this script with the output of the DPDK symbol tool, to notify maintainers
+of expired symbols by email. You need to define the environment variable
+DPDK_GETMAINTAINER_PATH, for this tool to work.
+
+Use terminal output to review the emails before sending.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+{s} --format-output terminal
+
+Then use email output to send the emails to the maintainers.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+{s} --format-output email \\
+--smtp-server <server> --sender <someone@somewhere.com> --password <password>
+'''
+
+EMAIL_TEMPLATE = '''Hi there,
+
+Please note the symbols listed below have expired. In line with the DPDK ABI
+policy, they should be scheduled for removal, in the next DPDK release.
+
+For more information, please see the DPDK ABI Policy, section 3.5.3.
+https://doc.dpdk.org/guides/contributing/abi_policy.html
+
+Thanks,
+
+The DPDK Symbol Bot
+
+'''
+
+ABI_POLICY = 'doc/guides/contributing/abi_policy.rst'
+get_maintainer = ['devtools/get-maintainer.sh', \
+ '--email', '-f']
+
+def _get_maintainers(libpath):
+ '''Get the maintainers for given library'''
+ try:
+ cmd = get_maintainer + [libpath]
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ return None
+
+ if result is None:
+ return None
+
+ email = result.stdout.decode('utf-8')
+ if email == '':
+ return None
+
+ email = list(filter(None,email.split('\n')))
+ return email
+
+default_maintainers = _get_maintainers(ABI_POLICY)
+
+def get_maintainers(libpath):
+ '''Get the maintainers for given library'''
+ maintainers=_get_maintainers(libpath)
+
+ if maintainers is None:
+ maintainers = default_maintainers
+
+ return maintainers
+
+def get_message(library, symbols):
+ '''Build email message from symbols, config and maintainers'''
+ message = {}
+ maintainers = get_maintainers(library)
+
+ message['To'] = maintainers
+ if maintainers != default_maintainers:
+ message['CC'] = default_maintainers
+
+ message['Subject'] = 'Expired symbols in {}\n'.format(library)
+
+ body = EMAIL_TEMPLATE
+ for sym in symbols:
+ body += ('{}\n'.format(sym))
+
+ message['Body'] = body
+
+ return message
+
+class OutputEmail():
+ '''Format the output for email'''
+ def __init__(self, config):
+ self.config = config
+
+ self.terminal = OutputTerminal(config)
+ context = ssl.create_default_context()
+
+ # Try to log in to server and send email
+ try:
+ self.server = smtplib.SMTP(config['smtp_server'], 587)
+ self.server.starttls(context=context) # Secure the connection
+ self.server.login(config['sender'], config['password'])
+ except Exception as exception:
+ print(exception)
+ raise exception
+
+ def message(self,message):
+ '''send email'''
+ self.terminal.message(message)
+
+ msg = EmailMessage()
+ msg.set_content(message.pop('Body'))
+
+ for key in message.keys():
+ msg[key] = message[key]
+
+ msg['From'] = self.config['sender']
+ msg['Reply-To'] = 'no-reply@dpdk.org'
+
+ self.server.send_message(msg)
+
+ time.sleep(1)
+
+ def __del__(self):
+ self.server.quit()
+
+class OutputTerminal(): # pylint: disable=too-few-public-methods
+ '''Format the output for the terminal'''
+ def __init__(self, config):
+ self.config = config
+
+ def message(self,message):
+ '''Print email to terminal'''
+ terminal = 'To:' + ', '.join(message['To']) + '\n'
+ if 'sender' in self.config.keys():
+ terminal += 'From:' + self.config['sender'] + '\n'
+
+ terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n'
+ if 'CC' in message.keys():
+ terminal += 'CC:' + ', '.join(message['CC']) + '\n'
+
+ terminal += 'Subject:' + message['Subject'] + '\n'
+ terminal += 'Body:' + message['Body'] + '\n'
+
+ print(terminal)
+ print('-' * 80)
+
+def parse_config(args):
+ '''put the command line args in the right places'''
+ config = {}
+ error_msg = None
+
+ outputs = {
+ None : OutputTerminal,
+ 'terminal' : OutputTerminal,
+ 'email' : OutputEmail
+ }
+
+ if args.format_output == 'email':
+ if args.smtp_server is None:
+ error_msg = 'SMTP server'
+ else:
+ config['smtp_server'] = args.smtp_server
+
+ if args.sender is None:
+ error_msg = 'sender'
+ else:
+ config['sender'] = args.sender
+
+ if args.password is None:
+ error_msg = 'password'
+ else:
+ config['password'] = args.password
+
+ if error_msg is not None:
+ print('Please specify a {} for email output'.format(error_msg))
+ return None
+
+ config['output'] = outputs[args.format_output]
+ return config
+
+def main():
+ '''Main entry point'''
+ parser = argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), \
+ formatter_class=RawTextHelpFormatter)
+ parser.add_argument('--format-output', choices=['terminal','email'], \
+ default='terminal')
+ parser.add_argument('--smtp-server')
+ parser.add_argument('--password')
+ parser.add_argument('--sender')
+
+ args = parser.parse_args()
+ config = parse_config(args)
+ if config is None:
+ return
+
+ symbols = []
+ lastlib = library = ''
+
+ output = config['output'](config)
+
+ for line in sys.stdin:
+ line = line.rstrip('\n')
+ library, symbol = [line[:line.find(',')], \
+ line[line.find(',') + 1: len(line)]]
+ if library == 'mapfile':
+ continue
+
+ if library != lastlib:
+ message = get_message(lastlib, symbols)
+ output.message(message)
+ symbols = []
+
+ lastlib = library
+ symbols = symbols + [symbol]
+
+ #print the last library
+ message = get_message(lastlib, symbols)
+ output.message(message)
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
` (7 preceding siblings ...)
2021-08-09 12:53 ` [dpdk-dev] [PATCH v9 0/2] devtools: scripts to count and track symbols Ray Kinsella
@ 2021-08-31 14:50 ` Ray Kinsella
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 1/3] devtools: script to track symbols over releases Ray Kinsella
` (3 more replies)
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 " Ray Kinsella
` (3 subsequent siblings)
12 siblings, 4 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-08-31 14:50 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr, aconole
Scripts to count and track the lifecycle of DPDK symbols.
The symbol-tool script reports on the growth of symbols over releases
and list expired symbols. The notify-symbol-maintainers script
consumes the input from symbol-tool and generates email notifications
of expired symbols.
v2: reworked to fix pylint errors
v3: sent with the correct in-reply-to
v4: fix typos picked up by the CI
v5: fix terminal_size & directory args
v6: added list-expired, to list expired experimental symbols
v7: fix typo in comments
v8: added tool to notify maintainers of expired symbols
v9: removed hardcoded emails addressed and script names
v10: added ability to identify and notify the original contributors
Ray Kinsella (3):
devtools: script to track symbols over releases
devtools: script to send notifications of expired symbols
maintainers: add new abi scripts
MAINTAINERS | 2 +
devtools/notify-symbol-maintainers.py | 256 ++++++++++++++
devtools/symbol-tool.py | 482 ++++++++++++++++++++++++++
3 files changed, 740 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
create mode 100755 devtools/symbol-tool.py
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v10 1/3] devtools: script to track symbols over releases
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols Ray Kinsella
@ 2021-08-31 14:50 ` Ray Kinsella
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 2/3] devtools: script to send notifications of expired symbols Ray Kinsella
` (2 subsequent siblings)
3 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-08-31 14:50 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr, aconole
This script tracks the growth of stable and experimental symbols
over releases since v19.11. The script has the ability to
count the added symbols between two dpdk releases, and to
list experimental symbols present in two dpdk releases
(expired symbols).
example usages:
Count symbols added since v19.11
$ devtools/symbol-tool.py count-symbols
Count symbols added since v20.11
$ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05
List experimental symbols present in v20.11 and v21.05
$ devtools/symbol-tool.py list-expired --releases v20.11,v21.05
List experimental symbols in libraries only, present since v19.11
$ devtools/symbol-tool.py list-expired --directory lib
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/symbol-tool.py | 482 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 482 insertions(+)
create mode 100755 devtools/symbol-tool.py
diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py
new file mode 100755
index 0000000000..3d093a0802
--- /dev/null
+++ b/devtools/symbol-tool.py
@@ -0,0 +1,482 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+# pylint: disable=invalid-name
+'''Tool to count or list symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import re
+import datetime
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ sys.exit()
+
+DESCRIPTION = '''
+This script tracks the growth of stable and experimental symbols
+over releases since v19.11. The script has the ability to
+count the added symbols between two dpdk releases, and to
+list experimental symbols present in two dpdk releases
+(expired symbols), including the name & email of the original contributor.
+
+example usages:
+
+Count symbols added since v19.11
+$ {s} count-symbols
+
+Count symbols added since v20.11
+$ {s} count-symbols --releases v20.11,v21.05
+
+List experimental symbols present in v20.11 and v21.05
+$ {s} list-expired --releases v20.11,v21.05
+
+List experimental symbols in libraries only, present since v19.11
+$ {s} list-expired --directory lib
+'''
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+"""
+
+def get_abi_versions():
+ '''Returns a string of possible dpdk abi versions'''
+
+ year = datetime.date.today().year - 2000
+ tags = " |".join(['\'{}\''.format(i) \
+ for i in reversed(range(21, year + 1)) ])
+ tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return tags
+
+def get_dpdk_releases():
+ '''Returns a list of dpdk release tags names since v19.11'''
+
+ year = datetime.date.today().year - 2000
+ year_range = "|".join("{}".format(i) for i in range(19,year + 1))
+ pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ try:
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ print("Failed to interogate git for release tags")
+ sys.exit()
+
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [ tag.replace('\"','') \
+ for tag in reversed(tags) \
+ if pattern.match(tag) ][:-3]
+
+ return tags
+
+def fix_directory_name(path):
+ '''Prepend librte to the source directory name'''
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+def directory_renamed(path, rel):
+ '''Fix removal of the librte_ from the directory names'''
+
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ return result
+
+def mapfile_renamed(path, rel):
+ '''Fix renaming of the map file'''
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree', \
+ rel, str(path.parent) + '/'], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if newfile is not None:
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+def mapfile_and_directory_renamed(path, rel):
+ '''Fix renaming of the map file & the source directory'''
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+FIX_STRATEGIES = [directory_renamed, \
+ mapfile_renamed, \
+ mapfile_and_directory_renamed]
+
+def get_symbols(map_parser, release, mapfile_path):
+ '''Count the symbols for a given release and mapfile'''
+ abi_sections = {}
+
+ tagfile = '{}:{}'.format(release,mapfile_path)
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ for fix_strategy in FIX_STRATEGIES:
+ if result is not None:
+ break
+ result = fix_strategy(mapfile_path, release)
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = map_parser(mapfile).abi()
+
+ return abi_sections
+
+def get_terminal_rows():
+ '''Find the number of rows in the terminal'''
+
+ try:
+ return os.get_terminal_size().lines
+ except IOError:
+ return 0
+
+class SymbolOwner():
+ '''Find the symbols original contributors name and email'''
+ symbol_regex = {}
+ blame_regex = {'name' : r'author\s(.*)', \
+ 'email' : r'author-mail\s<(.*)>'}
+
+ def __init__(self, libpath, symbol):
+ self.libpath = libpath
+ self.symbol = symbol
+
+ #find variable definitions in C files, and functions in headers.
+ self.symbol_regex = \
+ {'*.c' : r'^(?!extern).*' + self.symbol + '[^()]*;', \
+ '*.h' : r'__rte_experimental(?:.*\n){0,2}.*' + self.symbol}
+
+ def find_symbol_location(self):
+ '''Find where the symbol is definited in the source'''
+ for key in self.symbol_regex:
+ for path in Path(self.libpath).rglob(key):
+ file_text = open(path).read()
+
+ #find where the symbol is defined, either preceeded by
+ #rte_experimental tag (functions) or followed by a ; (variables)
+
+ exp = self.symbol_regex[key]
+ pattern = re.compile(exp, re.MULTILINE)
+ search = pattern.search(file_text)
+
+ if search is not None:
+ symbol_pos = search.span()[1]
+ symbol_line = file_text.count('\n', 0, symbol_pos) + 1
+
+ return [str(path),symbol_line]
+ return None
+
+ def find_symbol_owner(self):
+ '''Find the symbols original contributors name and email'''
+ owners = {}
+ location = self.find_symbol_location()
+
+ if location is None:
+ return None
+
+ line = '-L {},{}'.format(location[1],location[1])
+ #git blame -p(orcelain) -L(ine) path
+ args = ['-p', line, location[0]]
+
+ try:
+ result = subprocess.run(['git', 'blame'] + args, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ return None
+
+ blame = result.stdout.decode('utf-8')
+ for key in self.blame_regex:
+ pattern = re.compile(self.blame_regex[key], re.MULTILINE)
+ match = pattern.search(blame)
+
+ owners[key] = match.groups()[0]
+
+ return owners
+
+class SymbolCountOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + dpdk_releases
+
+ self.terminal_rows = get_terminal_rows()
+ self.row = 0
+
+ def set_terminal_output(self,dpdk_rel):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}' + \
+ ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+ self.column_fmt = '{:50}' + \
+ ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+ def set_csv_output(self,dpdk_rel):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},' + \
+ ','.join(['{},{}'] * (len(dpdk_rel)))
+ self.column_fmt = '{},' + \
+ ','.join(['{},'] * (len(dpdk_rel)))
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+ self.row += 1
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+ mapfile = str(mapfile)
+ print(self.output_fmt.format(*([mapfile] + symbols)))
+ self.row += 1
+
+ if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)):
+ self.print_columns()
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+class ListExpiredOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.terminal = True
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + \
+ ['expired (' + ','.join(dpdk_releases) + ')'] + \
+ ['contributor name', 'contributor email']
+
+ def set_terminal_output(self, _):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}{:<50}{:<25}{:<25}'
+ self.column_fmt = '{:50}{:50}{:25}{:25}'
+
+ def set_csv_output(self, _):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},{},{},{}'
+ self.column_fmt = '{},{},{},{}'
+ self.terminal = False
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+
+ def print_row(self, mapfile, symbols, owner):
+ '''Print row of symbol values'''
+
+ for symbol in symbols:
+ mapfile = str(mapfile)
+ name = owner[symbol]['name'] \
+ if owner[symbol] is not None else ''
+ email = owner[symbol]['email'] \
+ if owner[symbol] is not None else ''
+
+ print(self.output_fmt.format(mapfile, symbol, name, email))
+ if self.terminal:
+ mapfile = ''
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+class CountSymbolsAction:
+ ''' Logic to count symbols added since a give release '''
+ IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL']
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.symbols_count = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbol_count = experimental_count = 0
+
+ symbols = get_symbols(self.parser, release, self.path)
+
+ # which versions are present, and we care about
+ abi_vers = [abi_ver \
+ for abi_ver in symbols \
+ if abi_ver not in self.IGNORE_SECTIONS]
+
+ for abi_ver in abi_vers:
+ symbol_count += len(symbols[abi_ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental_count = len(symbols['EXPERIMENTAL'])
+
+ self.symbols_count += [symbol_count, experimental_count]
+
+ def __del__(self):
+ self.format_output.print_row(self.path.parent, self.symbols_count)
+
+class ListExpiredAction:
+ ''' Logic to list expired symbols between two releases '''
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.experimental_symbols = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbols = get_symbols(self.parser, release, self.path)
+
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental = [exp.strip() for exp in symbols['EXPERIMENTAL']]
+
+ self.experimental_symbols.append(experimental)
+
+ def __del__(self):
+ if len(self.experimental_symbols) != 2:
+ return
+
+ tmp = self.experimental_symbols
+ # find symbols present in both dpdk releases
+ intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]]
+
+ # check for empty set
+ if intersect_syms == []:
+ return
+
+ sym_owner = {}
+ for sym in intersect_syms:
+ sym_owner[sym] = SymbolOwner(self.path.parent, sym).find_symbol_owner()
+
+ self.format_output.print_row(self.path.parent, intersect_syms, sym_owner)
+
+SRC_DIRECTORIES = 'drivers,lib'
+
+ACTIONS = {None: CountSymbolsAction, \
+ 'count-symbols': CountSymbolsAction, \
+ 'list-expired': ListExpiredAction}
+
+ACTION_OUTPUT = {None: SymbolCountOutput, \
+ 'count-symbols': SymbolCountOutput, \
+ 'list-expired': ListExpiredOutput}
+
+def main():
+ '''Main entry point'''
+
+ dpdk_releases = get_dpdk_releases()
+
+ parser = argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), \
+ formatter_class=RawTextHelpFormatter
+ )
+ parser.add_argument('mode', choices=['count-symbols','list-expired'])
+ parser.add_argument('--format-output', choices=['terminal','csv'], \
+ default='terminal')
+ parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','),
+ default=SRC_DIRECTORIES)
+ parser.add_argument('--releases', \
+ help='2 x comma separated release tags e.g. \'' \
+ + ','.join([dpdk_releases[0],dpdk_releases[-1]]) \
+ + '\'')
+ args = parser.parse_args()
+
+ if args.releases is not None:
+ dpdk_releases = args.releases.split(',')
+
+ if args.mode == 'list-expired':
+ if len(dpdk_releases) < 2:
+ sys.exit('Please specify two releases to compare ' \
+ 'in \'list-expired\' mode.')
+ dpdk_releases = [dpdk_releases[0], dpdk_releases[len(dpdk_releases) - 1]]
+
+ action = ACTIONS[args.mode]
+ format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases)
+
+ map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+ map_parser = makeGrammar(map_grammar, {})
+
+ format_output.print_columns()
+
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ release_action = action(path, map_parser, format_output)
+
+ for release in dpdk_releases:
+ release_action.add_mapfile(release)
+
+ # all the magic happens in the destructor
+ del release_action
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v10 2/3] devtools: script to send notifications of expired symbols
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols Ray Kinsella
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 1/3] devtools: script to track symbols over releases Ray Kinsella
@ 2021-08-31 14:50 ` Ray Kinsella
2021-09-01 12:46 ` Aaron Conole
2021-09-01 13:01 ` David Marchand
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 3/3] maintainers: add new abi scripts Ray Kinsella
2021-09-01 12:31 ` [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols Aaron Conole
3 siblings, 2 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-08-31 14:50 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr, aconole
Use this script with the output of the DPDK symbol tool, to notify
maintainers of expired symbols by email. You need to define the environment
variable DPDK_GETMAINTAINER_PATH for this tool to work.
Use terminal output to review the emails before sending.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
devtools/notify_expired_symbols.py --format-output terminal
Then use email output to send the emails to the maintainers.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
devtools/notify_expired_symbols.py --format-output email \
--smtp-server <server> --sender <someone@somewhere.com> \
--password <password> --cc <someone@somewhere.com>
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/notify-symbol-maintainers.py | 256 ++++++++++++++++++++++++++
1 file changed, 256 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py
new file mode 100755
index 0000000000..ee554687ff
--- /dev/null
+++ b/devtools/notify-symbol-maintainers.py
@@ -0,0 +1,256 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+# pylint: disable=invalid-name
+'''Tool to notify maintainers of expired symbols'''
+import smtplib
+import ssl
+import sys
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import time
+from email.message import EmailMessage
+
+DESCRIPTION = '''
+Use this script with the output of the DPDK symbol tool, to notify maintainers
+and contributors of expired symbols by email. You need to define the environment
+variable DPDK_GETMAINTAINER_PATH for this tool to work.
+
+Use terminal output to review the emails before sending.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+{s} --format-output terminal
+
+Then use email output to send the emails to the maintainers.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+{s} --format-output email \\
+--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\
+--cc <someone@somewhere.com>
+'''
+
+EMAIL_TEMPLATE = '''Hi there,
+
+Please note the symbols listed below have expired. In line with the DPDK ABI
+policy, they should be scheduled for removal, in the next DPDK release.
+
+For more information, please see the DPDK ABI Policy, section 3.5.3.
+https://doc.dpdk.org/guides/contributing/abi_policy.html
+
+Thanks,
+
+The DPDK Symbol Bot
+
+'''
+
+ABI_POLICY = 'doc/guides/contributing/abi_policy.rst'
+MAINTAINERS = 'MAINTAINERS'
+get_maintainer = ['devtools/get-maintainer.sh', \
+ '--email', '-f']
+
+def _get_maintainers(libpath):
+ '''Get the maintainers for given library'''
+ try:
+ cmd = get_maintainer + [libpath]
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ return None
+
+ if result is None:
+ return None
+
+ email = result.stdout.decode('utf-8')
+ if email == '':
+ return None
+
+ email = list(filter(None,email.split('\n')))
+ return email
+
+default_maintainers = _get_maintainers(ABI_POLICY) + \
+ _get_maintainers(MAINTAINERS)
+
+def get_maintainers(libpath):
+ '''Get the maintainers for given library'''
+ maintainers=_get_maintainers(libpath)
+
+ if maintainers is None:
+ maintainers = default_maintainers
+
+ return maintainers
+
+def get_message(library, symbols, config):
+ '''Build email message from symbols, config and maintainers'''
+ contributors = {}
+ message = {}
+ maintainers = get_maintainers(library)
+
+ if maintainers != default_maintainers:
+ message['CC'] = default_maintainers.copy()
+
+ if 'CC' in config:
+ message.setdefault('CC',[]).append(config['CC'])
+
+ message['Subject'] = 'Expired symbols in {}\n'.format(library)
+
+ body = EMAIL_TEMPLATE
+ body += '{:<50}{:<25}{:<25}\n'.format('Symbol','Contributor','Email')
+ for sym in symbols:
+ body += ('{:<50}{:<25}{:<25}\n'.format(sym,\
+ symbols[sym]['name'],
+ symbols[sym]['email'],
+ ))
+ email = symbols[sym]['email']
+ contributors[email] = ''
+
+ contributors = list(contributors.keys())
+
+ message['To'] = maintainers + contributors
+ message['Body'] = body
+
+ return message
+
+class OutputEmail():
+ '''Format the output for email'''
+ def __init__(self, config):
+ self.config = config
+
+ self.terminal = OutputTerminal(config)
+ context = ssl.create_default_context()
+
+ # Try to log in to server and send email
+ try:
+ self.server = smtplib.SMTP(config['smtp_server'], 587)
+ self.server.starttls(context=context) # Secure the connection
+ self.server.login(config['sender'], config['password'])
+ except Exception as exception:
+ print(exception)
+ raise exception
+
+ def message(self,message):
+ '''send email'''
+ self.terminal.message(message)
+
+ msg = EmailMessage()
+ msg.set_content(message.pop('Body'))
+
+ for key in message.keys():
+ msg[key] = message[key]
+
+ msg['From'] = self.config['sender']
+ msg['Reply-To'] = 'no-reply@dpdk.org'
+
+ self.server.send_message(msg)
+
+ time.sleep(1)
+
+ def __del__(self):
+ self.server.quit()
+
+class OutputTerminal(): # pylint: disable=too-few-public-methods
+ '''Format the output for the terminal'''
+ def __init__(self, config):
+ self.config = config
+
+ def message(self,message):
+ '''Print email to terminal'''
+
+ terminal = 'To:' + ', '.join(message['To']) + '\n'
+ if 'sender' in self.config.keys():
+ terminal += 'From:' + self.config['sender'] + '\n'
+
+ terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n'
+
+ if 'CC' in message:
+ terminal += 'CC:' + ', '.join(message['CC']) + '\n'
+
+ terminal += 'Subject:' + message['Subject'] + '\n'
+ terminal += 'Body:' + message['Body'] + '\n'
+
+ print(terminal)
+ print('-' * 80)
+
+def parse_config(args):
+ '''put the command line args in the right places'''
+ config = {}
+ error_msg = None
+
+ outputs = {
+ None : OutputTerminal,
+ 'terminal' : OutputTerminal,
+ 'email' : OutputEmail
+ }
+
+ if args.format_output == 'email':
+ if args.smtp_server is None:
+ error_msg = 'SMTP server'
+ else:
+ config['smtp_server'] = args.smtp_server
+
+ if args.sender is None:
+ error_msg = 'sender'
+ else:
+ config['sender'] = args.sender
+
+ if args.password is None:
+ error_msg = 'password'
+ else:
+ config['password'] = args.password
+
+ if args.cc is not None:
+ config['CC'] = args.cc
+
+ if error_msg is not None:
+ print('Please specify a {} for email output'.format(error_msg))
+ return None
+
+ config['output'] = outputs[args.format_output]
+ return config
+
+def main():
+ '''Main entry point'''
+ parser = argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), \
+ formatter_class=RawTextHelpFormatter)
+ parser.add_argument('--format-output', choices=['terminal','email'], \
+ default='terminal')
+ parser.add_argument('--smtp-server')
+ parser.add_argument('--password')
+ parser.add_argument('--sender')
+ parser.add_argument('--cc')
+
+ args = parser.parse_args()
+ config = parse_config(args)
+ if config is None:
+ return
+
+ symbols = {}
+ lastlib = library = ''
+
+ output = config['output'](config)
+
+ for line in sys.stdin:
+ line = line.rstrip('\n')
+
+ if line.find('mapfile') >= 0:
+ continue
+ library, symbol, name, email = line.split(',')
+
+ if library != lastlib:
+ message = get_message(lastlib, symbols, config)
+ output.message(message)
+ symbols = {}
+
+ lastlib = library
+ symbols[symbol] = {'name' : name, 'email' : email}
+
+ #print the last library
+ message = get_message(lastlib, symbols, config)
+ output.message(message)
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v10 3/3] maintainers: add new abi scripts
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols Ray Kinsella
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 1/3] devtools: script to track symbols over releases Ray Kinsella
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 2/3] devtools: script to send notifications of expired symbols Ray Kinsella
@ 2021-08-31 14:50 ` Ray Kinsella
2021-09-01 12:31 ` [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols Aaron Conole
3 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-08-31 14:50 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr, aconole
Add new abi management scripts to the MAINTAINERS file.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
MAINTAINERS | 2 ++
1 file changed, 2 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 266f5ac1da..ff8245271f 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -129,6 +129,8 @@ F: devtools/gen-abi.sh
F: devtools/libabigail.abignore
F: devtools/update-abi.sh
F: devtools/update_version_map_abi.py
+F: devtools/notify-symbol-maintainers.py
+F: devtools/symbol-tool.py
F: buildtools/check-symbols.sh
F: buildtools/map-list-symbol.sh
F: drivers/*/*/*.map
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols Ray Kinsella
` (2 preceding siblings ...)
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 3/3] maintainers: add new abi scripts Ray Kinsella
@ 2021-09-01 12:31 ` Aaron Conole
2021-09-01 17:17 ` Stephen Hemminger
3 siblings, 1 reply; 52+ messages in thread
From: Aaron Conole @ 2021-09-01 12:31 UTC (permalink / raw)
To: Ray Kinsella
Cc: dev, bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor
Ray Kinsella <mdr@ashroe.eu> writes:
> Scripts to count and track the lifecycle of DPDK symbols.
>
> The symbol-tool script reports on the growth of symbols over releases
> and list expired symbols. The notify-symbol-maintainers script
> consumes the input from symbol-tool and generates email notifications
> of expired symbols.
>
> v2: reworked to fix pylint errors
> v3: sent with the correct in-reply-to
> v4: fix typos picked up by the CI
> v5: fix terminal_size & directory args
> v6: added list-expired, to list expired experimental symbols
> v7: fix typo in comments
> v8: added tool to notify maintainers of expired symbols
> v9: removed hardcoded emails addressed and script names
> v10: added ability to identify and notify the original contributors
>
> Ray Kinsella (3):
> devtools: script to track symbols over releases
> devtools: script to send notifications of expired symbols
> maintainers: add new abi scripts
>
> MAINTAINERS | 2 +
> devtools/notify-symbol-maintainers.py | 256 ++++++++++++++
> devtools/symbol-tool.py | 482 ++++++++++++++++++++++++++
> 3 files changed, 740 insertions(+)
> create mode 100755 devtools/notify-symbol-maintainers.py
> create mode 100755 devtools/symbol-tool.py
I get a whole mess of flake8 issues from this series (mostly 'backslash
is redundant' and whitespace issues). I'm using flake8 because it
pretty well enforces PEP8 style guide. I would like to see it
addressed, but also I see that many of the python files in the DPDK tree
don't actually pass. Example::
$ flake8 ./usertools/dpdk-devbind.py | wc -l
34
$ flake8 ./usertools/dpdk-devbind.py | sed 's@./usertools/dpdk-devbind.py[:0-9]* @@' | sort -u
E128 continuation line under-indented for visual indent
E302 expected 2 blank lines, found 1
E305 expected 2 blank lines after class or function definition, found 1
E501 line too long (105 > 79 characters)
E501 line too long (80 > 79 characters)
E501 line too long (82 > 79 characters)
E501 line too long (83 > 79 characters)
E501 line too long (84 > 79 characters)
E501 line too long (85 > 79 characters)
E501 line too long (86 > 79 characters)
E501 line too long (91 > 79 characters)
E502 the backslash is redundant between brackets
E722 do not use bare 'except'
Looks like we repeat the same kinds of errors everywhere (this is on
multiple tools). Some of our in-tree python is better than others (like
app/test/autotest.py which only has 1 flake).
Maybe we can address this. Other comments inline on the patches.
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v10 2/3] devtools: script to send notifications of expired symbols
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 2/3] devtools: script to send notifications of expired symbols Ray Kinsella
@ 2021-09-01 12:46 ` Aaron Conole
2021-09-03 11:15 ` Kinsella, Ray
2021-09-03 13:32 ` Kinsella, Ray
2021-09-01 13:01 ` David Marchand
1 sibling, 2 replies; 52+ messages in thread
From: Aaron Conole @ 2021-09-01 12:46 UTC (permalink / raw)
To: Ray Kinsella
Cc: dev, bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor
Ray Kinsella <mdr@ashroe.eu> writes:
> Use this script with the output of the DPDK symbol tool, to notify
> maintainers of expired symbols by email. You need to define the environment
> variable DPDK_GETMAINTAINER_PATH for this tool to work.
>
> Use terminal output to review the emails before sending.
> e.g.
> $ devtools/symbol-tool.py list-expired --format-output csv \
> | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
> devtools/notify_expired_symbols.py --format-output terminal
>
> Then use email output to send the emails to the maintainers.
> e.g.
> $ devtools/symbol-tool.py list-expired --format-output csv \
> | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
> devtools/notify_expired_symbols.py --format-output email \
> --smtp-server <server> --sender <someone@somewhere.com> \
> --password <password> --cc <someone@somewhere.com>
>
> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
> ---
> devtools/notify-symbol-maintainers.py | 256 ++++++++++++++++++++++++++
> 1 file changed, 256 insertions(+)
> create mode 100755 devtools/notify-symbol-maintainers.py
>
> diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py
> new file mode 100755
> index 0000000000..ee554687ff
> --- /dev/null
> +++ b/devtools/notify-symbol-maintainers.py
> @@ -0,0 +1,256 @@
> +#!/usr/bin/env python3
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2021 Intel Corporation
> +# pylint: disable=invalid-name
> +'''Tool to notify maintainers of expired symbols'''
> +import smtplib
> +import ssl
> +import sys
> +import subprocess
> +import argparse
> +from argparse import RawTextHelpFormatter
> +import time
> +from email.message import EmailMessage
> +
> +DESCRIPTION = '''
> +Use this script with the output of the DPDK symbol tool, to notify maintainers
> +and contributors of expired symbols by email. You need to define the environment
> +variable DPDK_GETMAINTAINER_PATH for this tool to work.
> +
> +Use terminal output to review the emails before sending.
> +e.g.
> +$ devtools/symbol-tool.py list-expired --format-output csv \\
> +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
> +{s} --format-output terminal
> +
> +Then use email output to send the emails to the maintainers.
> +e.g.
> +$ devtools/symbol-tool.py list-expired --format-output csv \\
> +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
> +{s} --format-output email \\
> +--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\
> +--cc <someone@somewhere.com>
> +'''
> +
> +EMAIL_TEMPLATE = '''Hi there,
> +
> +Please note the symbols listed below have expired. In line with the DPDK ABI
> +policy, they should be scheduled for removal, in the next DPDK release.
> +
> +For more information, please see the DPDK ABI Policy, section 3.5.3.
> +https://doc.dpdk.org/guides/contributing/abi_policy.html
> +
> +Thanks,
> +
> +The DPDK Symbol Bot
> +
> +'''
> +
> +ABI_POLICY = 'doc/guides/contributing/abi_policy.rst'
> +MAINTAINERS = 'MAINTAINERS'
> +get_maintainer = ['devtools/get-maintainer.sh', \
> + '--email', '-f']
Maybe it's best to make this something that can be overridden. There's
a series to change the .sh files to .py files. Perhaps an environment
variable or argument?
> +def _get_maintainers(libpath):
> + '''Get the maintainers for given library'''
> + try:
> + cmd = get_maintainer + [libpath]
> + result = subprocess.run(cmd, \
> + stdout=subprocess.PIPE, \
> + stderr=subprocess.PIPE,
> + check=True)
> + except subprocess.CalledProcessError:
> + return None
You might consider handling
except FileNotFoundError:
....
With a graceful exit and error message. In case the get_maintainers
path changes.
> + if result is None:
> + return None
> +
> + email = result.stdout.decode('utf-8')
> + if email == '':
> + return None
> +
> + email = list(filter(None,email.split('\n')))
> + return email
> +
> +default_maintainers = _get_maintainers(ABI_POLICY) + \
> + _get_maintainers(MAINTAINERS)
> +
> +def get_maintainers(libpath):
> + '''Get the maintainers for given library'''
> + maintainers=_get_maintainers(libpath)
> +
> + if maintainers is None:
> + maintainers = default_maintainers
> +
> + return maintainers
> +
> +def get_message(library, symbols, config):
> + '''Build email message from symbols, config and maintainers'''
> + contributors = {}
> + message = {}
> + maintainers = get_maintainers(library)
> +
> + if maintainers != default_maintainers:
> + message['CC'] = default_maintainers.copy()
> +
> + if 'CC' in config:
> + message.setdefault('CC',[]).append(config['CC'])
> +
> + message['Subject'] = 'Expired symbols in {}\n'.format(library)
> +
> + body = EMAIL_TEMPLATE
> + body += '{:<50}{:<25}{:<25}\n'.format('Symbol','Contributor','Email')
> + for sym in symbols:
> + body += ('{:<50}{:<25}{:<25}\n'.format(sym,\
> + symbols[sym]['name'],
> + symbols[sym]['email'],
> + ))
> + email = symbols[sym]['email']
> + contributors[email] = ''
> +
> + contributors = list(contributors.keys())
> +
> + message['To'] = maintainers + contributors
> + message['Body'] = body
> +
> + return message
> +
> +class OutputEmail():
> + '''Format the output for email'''
> + def __init__(self, config):
> + self.config = config
> +
> + self.terminal = OutputTerminal(config)
> + context = ssl.create_default_context()
> +
> + # Try to log in to server and send email
> + try:
> + self.server = smtplib.SMTP(config['smtp_server'], 587)
> + self.server.starttls(context=context) # Secure the connection
> + self.server.login(config['sender'], config['password'])
> + except Exception as exception:
> + print(exception)
> + raise exception
> +
> + def message(self,message):
> + '''send email'''
> + self.terminal.message(message)
> +
> + msg = EmailMessage()
> + msg.set_content(message.pop('Body'))
> +
> + for key in message.keys():
> + msg[key] = message[key]
> +
> + msg['From'] = self.config['sender']
> + msg['Reply-To'] = 'no-reply@dpdk.org'
> +
> + self.server.send_message(msg)
> +
> + time.sleep(1)
Why this sleep is needed?
> +
> + def __del__(self):
> + self.server.quit()
> +
> +class OutputTerminal(): # pylint: disable=too-few-public-methods
> + '''Format the output for the terminal'''
> + def __init__(self, config):
> + self.config = config
> +
> + def message(self,message):
> + '''Print email to terminal'''
> +
> + terminal = 'To:' + ', '.join(message['To']) + '\n'
> + if 'sender' in self.config.keys():
> + terminal += 'From:' + self.config['sender'] + '\n'
> +
> + terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n'
> +
> + if 'CC' in message:
> + terminal += 'CC:' + ', '.join(message['CC']) + '\n'
> +
> + terminal += 'Subject:' + message['Subject'] + '\n'
> + terminal += 'Body:' + message['Body'] + '\n'
> +
> + print(terminal)
> + print('-' * 80)
> +
> +def parse_config(args):
> + '''put the command line args in the right places'''
> + config = {}
> + error_msg = None
> +
> + outputs = {
> + None : OutputTerminal,
> + 'terminal' : OutputTerminal,
> + 'email' : OutputEmail
> + }
> +
> + if args.format_output == 'email':
> + if args.smtp_server is None:
> + error_msg = 'SMTP server'
> + else:
> + config['smtp_server'] = args.smtp_server
> +
> + if args.sender is None:
> + error_msg = 'sender'
> + else:
> + config['sender'] = args.sender
> +
> + if args.password is None:
> + error_msg = 'password'
> + else:
> + config['password'] = args.password
> +
> + if args.cc is not None:
> + config['CC'] = args.cc
> +
> + if error_msg is not None:
> + print('Please specify a {} for email output'.format(error_msg))
> + return None
> +
> + config['output'] = outputs[args.format_output]
> + return config
> +
> +def main():
> + '''Main entry point'''
> + parser = argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), \
> + formatter_class=RawTextHelpFormatter)
> + parser.add_argument('--format-output', choices=['terminal','email'], \
> + default='terminal')
> + parser.add_argument('--smtp-server')
> + parser.add_argument('--password')
> + parser.add_argument('--sender')
> + parser.add_argument('--cc')
> +
> + args = parser.parse_args()
> + config = parse_config(args)
> + if config is None:
> + return
> +
> + symbols = {}
> + lastlib = library = ''
> +
> + output = config['output'](config)
> +
> + for line in sys.stdin:
> + line = line.rstrip('\n')
> +
> + if line.find('mapfile') >= 0:
> + continue
> + library, symbol, name, email = line.split(',')
> +
> + if library != lastlib:
> + message = get_message(lastlib, symbols, config)
> + output.message(message)
> + symbols = {}
> +
> + lastlib = library
> + symbols[symbol] = {'name' : name, 'email' : email}
> +
> + #print the last library
> + message = get_message(lastlib, symbols, config)
> + output.message(message)
> +
> +if __name__ == '__main__':
> + main()
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v10 2/3] devtools: script to send notifications of expired symbols
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 2/3] devtools: script to send notifications of expired symbols Ray Kinsella
2021-09-01 12:46 ` Aaron Conole
@ 2021-09-01 13:01 ` David Marchand
2021-09-03 13:28 ` Kinsella, Ray
2021-09-03 13:34 ` Kinsella, Ray
1 sibling, 2 replies; 52+ messages in thread
From: David Marchand @ 2021-09-01 13:01 UTC (permalink / raw)
To: Ray Kinsella
Cc: dev, Bruce Richardson, Stephen Hemminger, Yigit, Ferruh,
Thomas Monjalon, Kevin Traynor, Aaron Conole
Hello Ray,
On Tue, Aug 31, 2021 at 4:51 PM Ray Kinsella <mdr@ashroe.eu> wrote:
>
> Use this script with the output of the DPDK symbol tool, to notify
> maintainers of expired symbols by email. You need to define the environment
> variable DPDK_GETMAINTAINER_PATH for this tool to work.
>
> Use terminal output to review the emails before sending.
Two comments:
- there are references of a previous name for the script,
%s/notify_expired_symbols.py/notify-symbol-maintainers.py/g
- and a reminder for the empty report that we received yesterday.
I think this can be reproduced with:
$ DPDK_GETMAINTAINER_PATH=devtools/get_maintainer.pl
devtools/notify-symbol-maintainers.py --format-output terminal <<EOF
> mapfile,expired (v21.08,v19.11),contributor name,contributor email
> lib/rib,rte_rib6_get_ip,Stephen Hemminger,stephen@networkplumber.org
> EOF
To:Ray Kinsella <mdr@ashroe.eu>, Thomas Monjalon <thomas@monjalon.net>
Reply-To:no-reply@dpdk.org
Subject:Expired symbols in
Body:Hi there,
Please note the symbols listed below have expired. In line with the DPDK ABI
policy, they should be scheduled for removal, in the next DPDK release.
For more information, please see the DPDK ABI Policy, section 3.5.3.
https://doc.dpdk.org/guides/contributing/abi_policy.html
Thanks,
The DPDK Symbol Bot
Symbol Contributor
Email
--------------------------------------------------------------------------------
^^^^
Here, empty report.
To:Vladimir Medvedkin <vladimir.medvedkin@intel.com>, stephen@networkplumber.org
Reply-To:no-reply@dpdk.org
CC:Ray Kinsella <mdr@ashroe.eu>, Thomas Monjalon <thomas@monjalon.net>
Subject:Expired symbols in lib/rib
Body:Hi there,
Please note the symbols listed below have expired. In line with the DPDK ABI
policy, they should be scheduled for removal, in the next DPDK release.
For more information, please see the DPDK ABI Policy, section 3.5.3.
https://doc.dpdk.org/guides/contributing/abi_policy.html
Thanks,
The DPDK Symbol Bot
Symbol Contributor
Email
rte_rib6_get_ip Stephen Hemminger
stephen@networkplumber.org
--------------------------------------------------------------------------------
--
David Marchand
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols
2021-09-01 12:31 ` [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols Aaron Conole
@ 2021-09-01 17:17 ` Stephen Hemminger
2021-09-01 19:04 ` Aaron Conole
0 siblings, 1 reply; 52+ messages in thread
From: Stephen Hemminger @ 2021-09-01 17:17 UTC (permalink / raw)
To: Aaron Conole
Cc: Ray Kinsella, dev, bruce.richardson, ferruh.yigit, thomas, ktraynor
On Wed, 01 Sep 2021 08:31:27 -0400
Aaron Conole <aconole@redhat.com> wrote:
> $ flake8 ./usertools/dpdk-devbind.py | sed 's@./usertools/dpdk-devbind.py[:0-9]* @@' | sort -u
> E128 continuation line under-indented for visual indent
> E302 expected 2 blank lines, found 1
> E305 expected 2 blank lines after class or function definition, found 1
> E501 line too long (105 > 79 characters)
> E501 line too long (80 > 79 characters)
> E501 line too long (82 > 79 characters)
> E501 line too long (83 > 79 characters)
> E501 line too long (84 > 79 characters)
> E501 line too long (85 > 79 characters)
> E501 line too long (86 > 79 characters)
> E501 line too long (91 > 79 characters)
Current practice on many projects has allowed lines up to 100 characters.
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols
2021-09-01 17:17 ` Stephen Hemminger
@ 2021-09-01 19:04 ` Aaron Conole
2021-09-03 11:17 ` Kinsella, Ray
0 siblings, 1 reply; 52+ messages in thread
From: Aaron Conole @ 2021-09-01 19:04 UTC (permalink / raw)
To: Stephen Hemminger
Cc: Ray Kinsella, dev, bruce.richardson, ferruh.yigit, thomas, ktraynor
Stephen Hemminger <stephen@networkplumber.org> writes:
> On Wed, 01 Sep 2021 08:31:27 -0400
> Aaron Conole <aconole@redhat.com> wrote:
>
>> $ flake8 ./usertools/dpdk-devbind.py | sed 's@./usertools/dpdk-devbind.py[:0-9]* @@' | sort -u
>> E128 continuation line under-indented for visual indent
>> E302 expected 2 blank lines, found 1
>> E305 expected 2 blank lines after class or function definition, found 1
>> E501 line too long (105 > 79 characters)
>> E501 line too long (80 > 79 characters)
>> E501 line too long (82 > 79 characters)
>> E501 line too long (83 > 79 characters)
>> E501 line too long (84 > 79 characters)
>> E501 line too long (85 > 79 characters)
>> E501 line too long (86 > 79 characters)
>> E501 line too long (91 > 79 characters)
>
> Current practice on many projects has allowed lines up to 100 characters.
It is probably okay to run with '--ignore=E501' (which will squelch the
character limit).
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v10 2/3] devtools: script to send notifications of expired symbols
2021-09-01 12:46 ` Aaron Conole
@ 2021-09-03 11:15 ` Kinsella, Ray
2021-09-03 13:32 ` Kinsella, Ray
1 sibling, 0 replies; 52+ messages in thread
From: Kinsella, Ray @ 2021-09-03 11:15 UTC (permalink / raw)
To: Aaron Conole
Cc: dev, bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor
On 01/09/2021 13:46, Aaron Conole wrote:
> Ray Kinsella <mdr@ashroe.eu> writes:
>
>> Use this script with the output of the DPDK symbol tool, to notify
>> maintainers of expired symbols by email. You need to define the environment
>> variable DPDK_GETMAINTAINER_PATH for this tool to work.
>>
>> Use terminal output to review the emails before sending.
>> e.g.
>> $ devtools/symbol-tool.py list-expired --format-output csv \
>> | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
>> devtools/notify_expired_symbols.py --format-output terminal
>>
>> Then use email output to send the emails to the maintainers.
>> e.g.
>> $ devtools/symbol-tool.py list-expired --format-output csv \
>> | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
>> devtools/notify_expired_symbols.py --format-output email \
>> --smtp-server <server> --sender <someone@somewhere.com> \
>> --password <password> --cc <someone@somewhere.com>
>>
>> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
>> ---
>> devtools/notify-symbol-maintainers.py | 256 ++++++++++++++++++++++++++
>> 1 file changed, 256 insertions(+)
>> create mode 100755 devtools/notify-symbol-maintainers.py
>>
>> diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py
>> new file mode 100755
>> index 0000000000..ee554687ff
>> --- /dev/null
>> +++ b/devtools/notify-symbol-maintainers.py
>> @@ -0,0 +1,256 @@
>> +#!/usr/bin/env python3
>> +# SPDX-License-Identifier: BSD-3-Clause
>> +# Copyright(c) 2021 Intel Corporation
>> +# pylint: disable=invalid-name
>> +'''Tool to notify maintainers of expired symbols'''
>> +import smtplib
>> +import ssl
>> +import sys
>> +import subprocess
>> +import argparse
>> +from argparse import RawTextHelpFormatter
>> +import time
>> +from email.message import EmailMessage
>> +
>> +DESCRIPTION = '''
>> +Use this script with the output of the DPDK symbol tool, to notify maintainers
>> +and contributors of expired symbols by email. You need to define the environment
>> +variable DPDK_GETMAINTAINER_PATH for this tool to work.
>> +
>> +Use terminal output to review the emails before sending.
>> +e.g.
>> +$ devtools/symbol-tool.py list-expired --format-output csv \\
>> +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
>> +{s} --format-output terminal
>> +
>> +Then use email output to send the emails to the maintainers.
>> +e.g.
>> +$ devtools/symbol-tool.py list-expired --format-output csv \\
>> +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
>> +{s} --format-output email \\
>> +--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\
>> +--cc <someone@somewhere.com>
>> +'''
>> +
>> +EMAIL_TEMPLATE = '''Hi there,
>> +
>> +Please note the symbols listed below have expired. In line with the DPDK ABI
>> +policy, they should be scheduled for removal, in the next DPDK release.
>> +
>> +For more information, please see the DPDK ABI Policy, section 3.5.3.
>> +https://doc.dpdk.org/guides/contributing/abi_policy.html
>> +
>> +Thanks,
>> +
>> +The DPDK Symbol Bot
>> +
>> +'''
>> +
>> +ABI_POLICY = 'doc/guides/contributing/abi_policy.rst'
>> +MAINTAINERS = 'MAINTAINERS'
>> +get_maintainer = ['devtools/get-maintainer.sh', \
>> + '--email', '-f']
>
> Maybe it's best to make this something that can be overridden. There's
> a series to change the .sh files to .py files. Perhaps an environment
> variable or argument?
ACK
>
>> +def _get_maintainers(libpath):
>> + '''Get the maintainers for given library'''
>> + try:
>> + cmd = get_maintainer + [libpath]
>> + result = subprocess.run(cmd, \
>> + stdout=subprocess.PIPE, \
>> + stderr=subprocess.PIPE,
>> + check=True)
>> + except subprocess.CalledProcessError:
>> + return None
>
> You might consider handling
>
> except FileNotFoundError:
> ....
>
> With a graceful exit and error message. In case the get_maintainers
> path changes.
ACK
>
>> + if result is None:
>> + return None
>> +
>> + email = result.stdout.decode('utf-8')
>> + if email == '':
>> + return None
>> +
>> + email = list(filter(None,email.split('\n')))
>> + return email
>> +
>> +default_maintainers = _get_maintainers(ABI_POLICY) + \
>> + _get_maintainers(MAINTAINERS)
>> +
>> +def get_maintainers(libpath):
>> + '''Get the maintainers for given library'''
>> + maintainers=_get_maintainers(libpath)
>> +
>> + if maintainers is None:
>> + maintainers = default_maintainers
>> +
>> + return maintainers
>> +
>> +def get_message(library, symbols, config):
>> + '''Build email message from symbols, config and maintainers'''
>> + contributors = {}
>> + message = {}
>> + maintainers = get_maintainers(library)
>> +
>> + if maintainers != default_maintainers:
>> + message['CC'] = default_maintainers.copy()
>> +
>> + if 'CC' in config:
>> + message.setdefault('CC',[]).append(config['CC'])
>> +
>> + message['Subject'] = 'Expired symbols in {}\n'.format(library)
>> +
>> + body = EMAIL_TEMPLATE
>> + body += '{:<50}{:<25}{:<25}\n'.format('Symbol','Contributor','Email')
>> + for sym in symbols:
>> + body += ('{:<50}{:<25}{:<25}\n'.format(sym,\
>> + symbols[sym]['name'],
>> + symbols[sym]['email'],
>> + ))
>> + email = symbols[sym]['email']
>> + contributors[email] = ''
>> +
>> + contributors = list(contributors.keys())
>> +
>> + message['To'] = maintainers + contributors
>> + message['Body'] = body
>> +
>> + return message
>> +
>> +class OutputEmail():
>> + '''Format the output for email'''
>> + def __init__(self, config):
>> + self.config = config
>> +
>> + self.terminal = OutputTerminal(config)
>> + context = ssl.create_default_context()
>> +
>> + # Try to log in to server and send email
>> + try:
>> + self.server = smtplib.SMTP(config['smtp_server'], 587)
>> + self.server.starttls(context=context) # Secure the connection
>> + self.server.login(config['sender'], config['password'])
>> + except Exception as exception:
>> + print(exception)
>> + raise exception
>> +
>> + def message(self,message):
>> + '''send email'''
>> + self.terminal.message(message)
>> +
>> + msg = EmailMessage()
>> + msg.set_content(message.pop('Body'))
>> +
>> + for key in message.keys():
>> + msg[key] = message[key]
>> +
>> + msg['From'] = self.config['sender']
>> + msg['Reply-To'] = 'no-reply@dpdk.org'
>> +
>> + self.server.send_message(msg)
>> +
>> + time.sleep(1)
>
> Why this sleep is needed?
Don't hammer the mail server :-)
>
>> +
>> + def __del__(self):
>> + self.server.quit()
>> +
>> +class OutputTerminal(): # pylint: disable=too-few-public-methods
>> + '''Format the output for the terminal'''
>> + def __init__(self, config):
>> + self.config = config
>> +
>> + def message(self,message):
>> + '''Print email to terminal'''
>> +
>> + terminal = 'To:' + ', '.join(message['To']) + '\n'
>> + if 'sender' in self.config.keys():
>> + terminal += 'From:' + self.config['sender'] + '\n'
>> +
>> + terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n'
>> +
>> + if 'CC' in message:
>> + terminal += 'CC:' + ', '.join(message['CC']) + '\n'
>> +
>> + terminal += 'Subject:' + message['Subject'] + '\n'
>> + terminal += 'Body:' + message['Body'] + '\n'
>> +
>> + print(terminal)
>> + print('-' * 80)
>> +
>> +def parse_config(args):
>> + '''put the command line args in the right places'''
>> + config = {}
>> + error_msg = None
>> +
>> + outputs = {
>> + None : OutputTerminal,
>> + 'terminal' : OutputTerminal,
>> + 'email' : OutputEmail
>> + }
>> +
>> + if args.format_output == 'email':
>> + if args.smtp_server is None:
>> + error_msg = 'SMTP server'
>> + else:
>> + config['smtp_server'] = args.smtp_server
>> +
>> + if args.sender is None:
>> + error_msg = 'sender'
>> + else:
>> + config['sender'] = args.sender
>> +
>> + if args.password is None:
>> + error_msg = 'password'
>> + else:
>> + config['password'] = args.password
>> +
>> + if args.cc is not None:
>> + config['CC'] = args.cc
>> +
>> + if error_msg is not None:
>> + print('Please specify a {} for email output'.format(error_msg))
>> + return None
>> +
>> + config['output'] = outputs[args.format_output]
>> + return config
>> +
>> +def main():
>> + '''Main entry point'''
>> + parser = argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), \
>> + formatter_class=RawTextHelpFormatter)
>> + parser.add_argument('--format-output', choices=['terminal','email'], \
>> + default='terminal')
>> + parser.add_argument('--smtp-server')
>> + parser.add_argument('--password')
>> + parser.add_argument('--sender')
>> + parser.add_argument('--cc')
>> +
>> + args = parser.parse_args()
>> + config = parse_config(args)
>> + if config is None:
>> + return
>> +
>> + symbols = {}
>> + lastlib = library = ''
>> +
>> + output = config['output'](config)
>> +
>> + for line in sys.stdin:
>> + line = line.rstrip('\n')
>> +
>> + if line.find('mapfile') >= 0:
>> + continue
>> + library, symbol, name, email = line.split(',')
>> +
>> + if library != lastlib:
>> + message = get_message(lastlib, symbols, config)
>> + output.message(message)
>> + symbols = {}
>> +
>> + lastlib = library
>> + symbols[symbol] = {'name' : name, 'email' : email}
>> +
>> + #print the last library
>> + message = get_message(lastlib, symbols, config)
>> + output.message(message)
>> +
>> +if __name__ == '__main__':
>> + main()
>
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols
2021-09-01 19:04 ` Aaron Conole
@ 2021-09-03 11:17 ` Kinsella, Ray
0 siblings, 0 replies; 52+ messages in thread
From: Kinsella, Ray @ 2021-09-03 11:17 UTC (permalink / raw)
To: Aaron Conole, Stephen Hemminger
Cc: dev, bruce.richardson, ferruh.yigit, thomas, ktraynor
On 01/09/2021 20:04, Aaron Conole wrote:
> Stephen Hemminger <stephen@networkplumber.org> writes:
>
>> On Wed, 01 Sep 2021 08:31:27 -0400
>> Aaron Conole <aconole@redhat.com> wrote:
>>
>>> $ flake8 ./usertools/dpdk-devbind.py | sed 's@./usertools/dpdk-devbind.py[:0-9]* @@' | sort -u
>>> E128 continuation line under-indented for visual indent
>>> E302 expected 2 blank lines, found 1
>>> E305 expected 2 blank lines after class or function definition, found 1
>>> E501 line too long (105 > 79 characters)
>>> E501 line too long (80 > 79 characters)
>>> E501 line too long (82 > 79 characters)
>>> E501 line too long (83 > 79 characters)
>>> E501 line too long (84 > 79 characters)
>>> E501 line too long (85 > 79 characters)
>>> E501 line too long (86 > 79 characters)
>>> E501 line too long (91 > 79 characters)
>>
>> Current practice on many projects has allowed lines up to 100 characters.
>
> It is probably okay to run with '--ignore=E501' (which will squelch the
> character limit).
I added # noqa : E501 in the appropriate places, as we can't depend on folks using --ignore=E501.
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v11 0/3] devtools: scripts to count and track symbols
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
` (8 preceding siblings ...)
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols Ray Kinsella
@ 2021-09-03 13:23 ` Ray Kinsella
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 1/3] devtools: script to track symbols over releases Ray Kinsella
` (2 more replies)
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 0/3] devtools: scripts to count and track symbols Ray Kinsella
` (2 subsequent siblings)
12 siblings, 3 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-03 13:23 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr, aconole
Scripts to count and track the lifecycle of DPDK symbols.
The symbol-tool script reports on the growth of symbols over releases
and list expired symbols. The notify-symbol-maintainers script
consumes the input from symbol-tool and generates email notifications
of expired symbols.
v2: reworked to fix pylint errors
v3: sent with the correct in-reply-to
v4: fix typos picked up by the CI
v5: fix terminal_size & directory args
v6: added list-expired, to list expired experimental symbols
v7: fix typo in comments
v8: added tool to notify maintainers of expired symbols
v9: removed hardcoded emails addressed and script names
v10: added ability to identify and notify the original contributors
v11: addressed feedback from Aaron Conole, including PEP8 errors.
Ray Kinsella (3):
devtools: script to track symbols over releases
devtools: script to send notifications of expired symbols
maintainers: add new abi scripts
MAINTAINERS | 2 +
devtools/notify-symbol-maintainers.py | 302 +++++++++++++++
devtools/symbol-tool.py | 505 ++++++++++++++++++++++++++
3 files changed, 809 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
create mode 100755 devtools/symbol-tool.py
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v11 1/3] devtools: script to track symbols over releases
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 " Ray Kinsella
@ 2021-09-03 13:23 ` Ray Kinsella
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 2/3] devtools: script to send notifications of expired symbols Ray Kinsella
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 3/3] maintainers: add new abi scripts Ray Kinsella
2 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-03 13:23 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr, aconole
This script tracks the growth of stable and experimental symbols
over releases since v19.11. The script has the ability to
count the added symbols between two dpdk releases, and to
list experimental symbols present in two dpdk releases
(expired symbols).
example usages:
Count symbols added since v19.11
$ devtools/symbol-tool.py count-symbols
Count symbols added since v20.11
$ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05
List experimental symbols present in v20.11 and v21.05
$ devtools/symbol-tool.py list-expired --releases v20.11,v21.05
List experimental symbols in libraries only, present since v19.11
$ devtools/symbol-tool.py list-expired --directory lib
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/symbol-tool.py | 505 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 505 insertions(+)
create mode 100755 devtools/symbol-tool.py
diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py
new file mode 100755
index 0000000000..a0b81c1b90
--- /dev/null
+++ b/devtools/symbol-tool.py
@@ -0,0 +1,505 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+# pylint: disable=invalid-name
+'''Tool to count or list symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import re
+import datetime
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ sys.exit()
+
+DESCRIPTION = '''
+This script tracks the growth of stable and experimental symbols
+over releases since v19.11. The script has the ability to
+count the added symbols between two dpdk releases, and to
+list experimental symbols present in two dpdk releases
+(expired symbols), including the name & email of the original contributor.
+
+example usages:
+
+Count symbols added since v19.11
+$ {s} count-symbols
+
+Count symbols added since v20.11
+$ {s} count-symbols --releases v20.11,v21.05
+
+List experimental symbols present in v20.11 and v21.05
+$ {s} list-expired --releases v20.11,v21.05
+
+List experimental symbols in libraries only, present since v19.11
+$ {s} list-expired --directory lib
+'''
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+""" # noqa: E501
+
+
+def get_abi_versions():
+ '''Returns a string of possible dpdk abi versions'''
+
+ year = datetime.date.today().year - 2000
+ tags = " |".join(['\'{}\''.format(i)
+ for i in reversed(range(21, year + 1))])
+ tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return tags
+
+
+def get_dpdk_releases():
+ '''Returns a list of dpdk release tags names since v19.11'''
+
+ year = datetime.date.today().year - 2000
+ year_range = "|".join("{}".format(i) for i in range(19, year + 1))
+ pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ try:
+ result = subprocess.run(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ print("Failed to interogate git for release tags")
+ sys.exit()
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [tag.replace('\"', '')
+ for tag in reversed(tags)
+ if pattern.match(tag)][:-3]
+
+ return tags
+
+
+def fix_directory_name(path):
+ '''Prepend librte to the source directory name'''
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+
+def directory_renamed(path, rel):
+ '''Fix removal of the librte_ from the directory names'''
+
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ return result
+
+
+def mapfile_renamed(path, rel):
+ '''Fix renaming of the map file'''
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree',
+ rel, str(path.parent) + '/'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if newfile is not None:
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+
+def mapfile_and_directory_renamed(path, rel):
+ '''Fix renaming of the map file & the source directory'''
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path), path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+
+FIX_STRATEGIES = [directory_renamed,
+ mapfile_renamed,
+ mapfile_and_directory_renamed]
+
+
+def get_symbols(map_parser, release, mapfile_path):
+ '''Count the symbols for a given release and mapfile'''
+ abi_sections = {}
+
+ tagfile = '{}:{}'.format(release, mapfile_path)
+ try:
+ result = subprocess.run(['git', 'show', tagfile],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ for fix_strategy in FIX_STRATEGIES:
+ if result is not None:
+ break
+ result = fix_strategy(mapfile_path, release)
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = map_parser(mapfile).abi()
+
+ return abi_sections
+
+
+def get_terminal_rows():
+ '''Find the number of rows in the terminal'''
+
+ try:
+ return os.get_terminal_size().lines
+ except IOError:
+ return 0
+
+
+class SymbolOwner():
+ '''Find the symbols original contributors name and email'''
+ symbol_regex = {}
+ blame_regex = {'name': r'author\s(.*)',
+ 'email': r'author-mail\s<(.*)>'}
+
+ def __init__(self, libpath, symbol):
+ self.libpath = libpath
+ self.symbol = symbol
+
+ # find variable definitions in C files, and functions in headers.
+ self.symbol_regex = \
+ {'*.c': r'^(?!extern).*' + self.symbol + '[^()]*;',
+ '*.h': r'__rte_experimental(?:.*\n){0,2}.*' + self.symbol}
+
+ def find_symbol_location(self):
+ '''Find where the symbol is definited in the source'''
+ for key in self.symbol_regex:
+ for path in Path(self.libpath).rglob(key):
+ file_text = open(path).read()
+
+ # find where the symbol is defined, either preceeded by
+ # rte_experimental tag (functions)
+ # or followed by a ; (variables)
+
+ exp = self.symbol_regex[key]
+ pattern = re.compile(exp, re.MULTILINE)
+ search = pattern.search(file_text)
+
+ if search is not None:
+ symbol_pos = search.span()[1]
+ symbol_line = file_text.count('\n', 0, symbol_pos) + 1
+
+ return [str(path), symbol_line]
+ return None
+
+ def find_symbol_owner(self):
+ '''Find the symbols original contributors name and email'''
+ owners = {}
+ location = self.find_symbol_location()
+
+ if location is None:
+ return None
+
+ line = '-L {},{}'.format(location[1], location[1])
+ # git blame -p(orcelain) -L(ine) path
+ args = ['-p', line, location[0]]
+
+ try:
+ result = subprocess.run(['git', 'blame'] + args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ return None
+
+ blame = result.stdout.decode('utf-8')
+ for key in self.blame_regex:
+ pattern = re.compile(self.blame_regex[key], re.MULTILINE)
+ match = pattern.search(blame)
+
+ owners[key] = match.groups()[0]
+
+ return owners
+
+
+class SymbolCountOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.OUTPUT_FORMATS[format_output](self, dpdk_releases)
+ self.column_titles = ['mapfile'] + dpdk_releases
+
+ self.terminal_rows = get_terminal_rows()
+ self.row = 0
+
+ def set_terminal_output(self, dpdk_rel):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}' + \
+ ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+ self.column_fmt = '{:50}' + \
+ ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+ def set_csv_output(self, dpdk_rel):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},' + \
+ ','.join(['{},{}'] * (len(dpdk_rel)))
+ self.column_fmt = '{},' + \
+ ','.join(['{},'] * (len(dpdk_rel)))
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+ self.row += 1
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+ mapfile = str(mapfile)
+ print(self.output_fmt.format(*([mapfile] + symbols)))
+ self.row += 1
+
+ if((self.terminal_rows > 0) and
+ ((self.row % self.terminal_rows) == 0)):
+ self.print_columns()
+
+ OUTPUT_FORMATS = {None: set_terminal_output,
+ 'terminal': set_terminal_output,
+ 'csv': set_csv_output}
+
+
+class ListExpiredOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.terminal = True
+ self.OUTPUT_FORMATS[format_output](self, dpdk_releases)
+ self.column_titles = ['mapfile'] + \
+ ['expired (' + ','.join(dpdk_releases) + ')'] + \
+ ['contributor name', 'contributor email']
+
+ def set_terminal_output(self, _):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}{:<50}{:<25}{:<25}'
+ self.column_fmt = '{:50}{:50}{:25}{:25}'
+
+ def set_csv_output(self, _):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},{},{},{}'
+ self.column_fmt = '{},{},{},{}'
+ self.terminal = False
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+
+ def print_row(self, mapfile, symbols, owner):
+ '''Print row of symbol values'''
+
+ for symbol in symbols:
+ mapfile = str(mapfile)
+ name = owner[symbol]['name'] \
+ if owner[symbol] is not None else ''
+ email = owner[symbol]['email'] \
+ if owner[symbol] is not None else ''
+
+ print(self.output_fmt.format(mapfile, symbol, name, email))
+ if self.terminal:
+ mapfile = ''
+
+ OUTPUT_FORMATS = {None: set_terminal_output,
+ 'terminal': set_terminal_output,
+ 'csv': set_csv_output}
+
+
+class CountSymbolsAction:
+ ''' Logic to count symbols added since a give release '''
+ IGNORE_SECTIONS = ['EXPERIMENTAL', 'INTERNAL']
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.symbols_count = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbol_count = experimental_count = 0
+
+ symbols = get_symbols(self.parser, release, self.path)
+
+ # which versions are present, and we care about
+ abi_vers = [abi_ver
+ for abi_ver in symbols
+ if abi_ver not in self.IGNORE_SECTIONS]
+
+ for abi_ver in abi_vers:
+ symbol_count += len(symbols[abi_ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental_count = len(symbols['EXPERIMENTAL'])
+
+ self.symbols_count += [symbol_count, experimental_count]
+
+ def __del__(self):
+ self.format_output.print_row(self.path.parent, self.symbols_count)
+
+
+class ListExpiredAction:
+ ''' Logic to list expired symbols between two releases '''
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.experimental_symbols = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbols = get_symbols(self.parser, release, self.path)
+
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental = [exp.strip() for exp in symbols['EXPERIMENTAL']]
+
+ self.experimental_symbols.append(experimental)
+
+ def __del__(self):
+ if len(self.experimental_symbols) != 2:
+ return
+
+ tmp = self.experimental_symbols
+ # find symbols present in both dpdk releases
+ intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]]
+
+ # check for empty set
+ if intersect_syms == []:
+ return
+
+ sym_owner = {}
+ for sym in intersect_syms:
+ sym_owner[sym] = \
+ SymbolOwner(self.path.parent, sym).find_symbol_owner()
+
+ self.format_output.print_row(self.path.parent,
+ intersect_syms,
+ sym_owner)
+
+
+SRC_DIRECTORIES = 'drivers,lib'
+
+ACTIONS = {None: CountSymbolsAction,
+ 'count-symbols': CountSymbolsAction,
+ 'list-expired': ListExpiredAction}
+
+ACTION_OUTPUT = {None: SymbolCountOutput,
+ 'count-symbols': SymbolCountOutput,
+ 'list-expired': ListExpiredOutput}
+
+
+def main():
+ '''Main entry point'''
+
+ dpdk_releases = get_dpdk_releases()
+
+ parser = \
+ argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__),
+ formatter_class=RawTextHelpFormatter)
+
+ parser.add_argument('mode', choices=['count-symbols', 'list-expired'])
+ parser.add_argument('--format-output', choices=['terminal', 'csv'],
+ default='terminal')
+ parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','),
+ default=SRC_DIRECTORIES)
+ parser.add_argument('--releases',
+ help='2 x comma separated release tags e.g. \''
+ + ','.join([dpdk_releases[0], dpdk_releases[-1]])
+ + '\'')
+ args = parser.parse_args()
+
+ if args.releases is not None:
+ dpdk_releases = args.releases.split(',')
+
+ if args.mode == 'list-expired':
+ if len(dpdk_releases) < 2:
+ sys.exit('Please specify two releases to compare '
+ 'in \'list-expired\' mode.')
+ dpdk_releases = [dpdk_releases[0],
+ dpdk_releases[len(dpdk_releases) - 1]]
+
+ action = ACTIONS[args.mode]
+ format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases)
+
+ map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+ map_parser = makeGrammar(map_grammar, {})
+
+ format_output.print_columns()
+
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ release_action = action(path, map_parser, format_output)
+
+ for release in dpdk_releases:
+ release_action.add_mapfile(release)
+
+ # all the magic happens in the destructor
+ del release_action
+
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v11 2/3] devtools: script to send notifications of expired symbols
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 " Ray Kinsella
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 1/3] devtools: script to track symbols over releases Ray Kinsella
@ 2021-09-03 13:23 ` Ray Kinsella
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 3/3] maintainers: add new abi scripts Ray Kinsella
2 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-03 13:23 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr, aconole
Use this script with the output of the DPDK symbol tool, to notify
maintainers of expired symbols by email. You need to define the environment
variable DPDK_GETMAINTAINER_PATH for this tool to work.
Use terminal output to review the emails before sending.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
devtools/notify_expired_symbols.py --format-output terminal
Then use email output to send the emails to the maintainers.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
devtools/notify_expired_symbols.py --format-output email \
--smtp-server <server> --sender <someone@somewhere.com> \
--password <password> --cc <someone@somewhere.com>
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/notify-symbol-maintainers.py | 302 ++++++++++++++++++++++++++
1 file changed, 302 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py
new file mode 100755
index 0000000000..edf330f88b
--- /dev/null
+++ b/devtools/notify-symbol-maintainers.py
@@ -0,0 +1,302 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+# pylint: disable=invalid-name
+'''Tool to notify maintainers of expired symbols'''
+import os
+import smtplib
+import ssl
+import sys
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import time
+from email.message import EmailMessage
+from pathlib import Path
+
+DESCRIPTION = '''
+Use this script with the output of the DPDK symbol tool, to notify maintainers
+and contributors of expired symbols by email. You need to define the environment
+variable DPDK_GETMAINTAINER_PATH for this tool to work.
+
+Use terminal output to review the emails before sending.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+{s} --format-output terminal
+
+Then use email output to send the emails to the maintainers.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+{s} --format-output email \\
+--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\
+--cc <someone@somewhere.com>
+''' # noqa: E501
+
+EMAIL_TEMPLATE = '''Hi there,
+
+Please note the symbols listed below have expired. In line with the DPDK ABI
+policy, they should be scheduled for removal, in the next DPDK release.
+
+For more information, please see the DPDK ABI Policy, section 3.5.3.
+https://doc.dpdk.org/guides/contributing/abi_policy.html
+
+Thanks,
+
+The DPDK Symbol Bot
+
+''' # noqa: E501
+
+ABI_POLICY = 'doc/guides/contributing/abi_policy.rst'
+DPDK_GMP_ENV_VAR = 'DPDK_GETMAINTAINER_PATH'
+MAINTAINERS = 'MAINTAINERS'
+get_maintainer = ['devtools/get-maintainer.sh',
+ '--email', '-f']
+
+
+class EnvironException(Exception):
+ '''Subclass exception for Pylint\'s happiness.'''
+
+
+def _die_on_exception(e):
+ '''Print an exception, and quit'''
+
+ print('Fatal Error: ' + str(e))
+ sys.exit()
+
+
+def _check_get_maintainers_env():
+ '''Check get maintainers scripts are setup'''
+
+ if not Path(get_maintainer[0]).is_file():
+ raise EnvironException('Cannot locate DPDK\'s get maintainers script, '
+ ' usually at $' + get_maintainer[0] + '.')
+
+ if DPDK_GMP_ENV_VAR not in os.environ:
+ raise EnvironException(DPDK_GMP_ENV_VAR + ' is not defined.')
+
+ if not Path(os.environ[DPDK_GMP_ENV_VAR]).is_file():
+ raise EnvironException('Cannot locate get maintainers script, usually'
+ ' at ' + DPDK_GMP_ENV_VAR + '.')
+
+
+def _get_maintainers(libpath):
+ '''Get the maintainers for given library'''
+
+ try:
+ _check_get_maintainers_env()
+ except EnvironException as e:
+ _die_on_exception(e)
+
+ try:
+ cmd = get_maintainer + [libpath]
+ result = subprocess.run(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError as e:
+ _die_on_exception(e)
+
+ if result is None:
+ return None
+
+ email = result.stdout.decode('utf-8')
+ if email == '':
+ return None
+
+ email = list(filter(None, email.split('\n')))
+ return email
+
+
+default_maintainers = _get_maintainers(ABI_POLICY) + \
+ _get_maintainers(MAINTAINERS)
+
+
+def get_maintainers(libpath):
+ '''Get the maintainers for given library'''
+ maintainers = _get_maintainers(libpath)
+
+ if maintainers is None:
+ maintainers = default_maintainers
+
+ return maintainers
+
+
+def get_message(library, symbols, config):
+ '''Build email message from symbols, config and maintainers'''
+ contributors = {}
+ message = {}
+ maintainers = get_maintainers(library)
+
+ if maintainers != default_maintainers:
+ message['CC'] = default_maintainers.copy()
+
+ if 'CC' in config:
+ message.setdefault('CC', []).append(config['CC'])
+
+ message['Subject'] = 'Expired symbols in {}\n'.format(library)
+
+ body = EMAIL_TEMPLATE
+ body += '{:<50}{:<25}{:<25}\n'.format('Symbol', 'Contributor', 'Email')
+ for sym in symbols:
+ body += ('{:<50}{:<25}{:<25}\n'.format(sym,
+ symbols[sym]['name'],
+ symbols[sym]['email']))
+ email = symbols[sym]['email']
+ contributors[email] = ''
+
+ contributors = list(contributors.keys())
+
+ message['To'] = maintainers + contributors
+ message['Body'] = body
+
+ return message
+
+
+class OutputEmail():
+ '''Format the output for email'''
+
+ def __init__(self, config):
+ self.config = config
+
+ self.terminal = OutputTerminal(config)
+ context = ssl.create_default_context()
+
+ # Try to log in to server and send email
+ try:
+ self.server = smtplib.SMTP(config['smtp_server'], 587)
+ self.server.starttls(context=context) # Secure the connection
+ self.server.login(config['sender'], config['password'])
+ except EnvironException as e:
+ _die_on_exception(e)
+
+ def message(self, message):
+ '''send email'''
+ self.terminal.message(message)
+
+ msg = EmailMessage()
+ msg.set_content(message.pop('Body'))
+
+ for key in message.keys():
+ msg[key] = message[key]
+
+ msg['From'] = self.config['sender']
+ msg['Reply-To'] = 'no-reply@dpdk.org'
+
+ self.server.send_message(msg)
+
+ time.sleep(1)
+
+ def __del__(self):
+ self.server.quit()
+
+
+class OutputTerminal(): # pylint: disable=too-few-public-methods
+ '''Format the output for the terminal'''
+
+ def __init__(self, config):
+ self.config = config
+
+ def message(self, message):
+ '''Print email to terminal'''
+
+ terminal = 'To:' + ', '.join(message['To']) + '\n'
+ if 'sender' in self.config.keys():
+ terminal += 'From:' + self.config['sender'] + '\n'
+
+ terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n'
+
+ if 'CC' in message:
+ terminal += 'CC:' + ', '.join(message['CC']) + '\n'
+
+ terminal += 'Subject:' + message['Subject'] + '\n'
+ terminal += 'Body:' + message['Body'] + '\n'
+
+ print(terminal)
+ print('-' * 80)
+
+
+def parse_config(args):
+ '''put the command line args in the right places'''
+ config = {}
+ error_msg = None
+
+ outputs = {
+ None: OutputTerminal,
+ 'terminal': OutputTerminal,
+ 'email': OutputEmail
+ }
+
+ if args.format_output == 'email':
+ if args.smtp_server is None:
+ error_msg = 'SMTP server'
+ else:
+ config['smtp_server'] = args.smtp_server
+
+ if args.sender is None:
+ error_msg = 'sender'
+ else:
+ config['sender'] = args.sender
+
+ if args.password is None:
+ error_msg = 'password'
+ else:
+ config['password'] = args.password
+
+ if args.cc is not None:
+ config['CC'] = args.cc
+
+ if error_msg is not None:
+ print('Please specify a {} for email output'.format(error_msg))
+ return None
+
+ config['output'] = outputs[args.format_output]
+ return config
+
+
+def main():
+ '''Main entry point'''
+ parser = \
+ argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__),
+ formatter_class=RawTextHelpFormatter)
+ parser.add_argument('--format-output',
+ choices=['terminal', 'email'],
+ default='terminal')
+ parser.add_argument('--smtp-server')
+ parser.add_argument('--password')
+ parser.add_argument('--sender')
+ parser.add_argument('--cc')
+
+ args = parser.parse_args()
+ config = parse_config(args)
+ if config is None:
+ return
+
+ symbols = {}
+ lastlib = library = ''
+
+ output = config['output'](config)
+
+ for line in sys.stdin:
+ line = line.rstrip('\n')
+
+ if line.find('mapfile') >= 0:
+ continue
+ library, symbol, name, email = line.split(',')
+
+ if library != lastlib:
+ message = get_message(lastlib, symbols, config)
+ output.message(message)
+ symbols = {}
+
+ lastlib = library
+ symbols[symbol] = {'name': name, 'email': email}
+
+ # print the last library
+ message = get_message(lastlib, symbols, config)
+ output.message(message)
+
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v11 3/3] maintainers: add new abi scripts
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 " Ray Kinsella
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 1/3] devtools: script to track symbols over releases Ray Kinsella
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 2/3] devtools: script to send notifications of expired symbols Ray Kinsella
@ 2021-09-03 13:23 ` Ray Kinsella
2 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-03 13:23 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr, aconole
Add new abi management scripts to the MAINTAINERS file.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
MAINTAINERS | 2 ++
1 file changed, 2 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 266f5ac1da..ff8245271f 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -129,6 +129,8 @@ F: devtools/gen-abi.sh
F: devtools/libabigail.abignore
F: devtools/update-abi.sh
F: devtools/update_version_map_abi.py
+F: devtools/notify-symbol-maintainers.py
+F: devtools/symbol-tool.py
F: buildtools/check-symbols.sh
F: buildtools/map-list-symbol.sh
F: drivers/*/*/*.map
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v10 2/3] devtools: script to send notifications of expired symbols
2021-09-01 13:01 ` David Marchand
@ 2021-09-03 13:28 ` Kinsella, Ray
2021-09-03 13:34 ` Kinsella, Ray
1 sibling, 0 replies; 52+ messages in thread
From: Kinsella, Ray @ 2021-09-03 13:28 UTC (permalink / raw)
To: David Marchand
Cc: dev, Bruce Richardson, Stephen Hemminger, Yigit, Ferruh,
Thomas Monjalon, Kevin Traynor, Aaron Conole
Hi David,
On 01/09/2021 14:01, David Marchand wrote:
> Hello Ray,
>
> On Tue, Aug 31, 2021 at 4:51 PM Ray Kinsella <mdr@ashroe.eu> wrote:
>>
>> Use this script with the output of the DPDK symbol tool, to notify
>> maintainers of expired symbols by email. You need to define the environment
>> variable DPDK_GETMAINTAINER_PATH for this tool to work.
>>
>> Use terminal output to review the emails before sending.
Just realized I missed this.
>
> Two comments:
> - there are references of a previous name for the script,
> %s/notify_expired_symbols.py/notify-symbol-maintainers.py/g
Fixed in v11 = I used __file__ instead.
> - and a reminder for the empty report that we received yesterday.
> I think this can be reproduced with:
Yes - I remember that, I will fix in v12.
>
> $ DPDK_GETMAINTAINER_PATH=devtools/get_maintainer.pl
> devtools/notify-symbol-maintainers.py --format-output terminal <<EOF
>> mapfile,expired (v21.08,v19.11),contributor name,contributor email
>> lib/rib,rte_rib6_get_ip,Stephen Hemminger,stephen@networkplumber.org
>> EOF
> To:Ray Kinsella <mdr@ashroe.eu>, Thomas Monjalon <thomas@monjalon.net>
> Reply-To:no-reply@dpdk.org
> Subject:Expired symbols in
>
> Body:Hi there,
>
> Please note the symbols listed below have expired. In line with the DPDK ABI
> policy, they should be scheduled for removal, in the next DPDK release.
>
> For more information, please see the DPDK ABI Policy, section 3.5.3.
> https://doc.dpdk.org/guides/contributing/abi_policy.html
>
> Thanks,
>
> The DPDK Symbol Bot
>
> Symbol Contributor
> Email
>
>
> --------------------------------------------------------------------------------
>
> ^^^^
> Here, empty report.
>
> To:Vladimir Medvedkin <vladimir.medvedkin@intel.com>, stephen@networkplumber.org
> Reply-To:no-reply@dpdk.org
> CC:Ray Kinsella <mdr@ashroe.eu>, Thomas Monjalon <thomas@monjalon.net>
> Subject:Expired symbols in lib/rib
>
> Body:Hi there,
>
> Please note the symbols listed below have expired. In line with the DPDK ABI
> policy, they should be scheduled for removal, in the next DPDK release.
>
> For more information, please see the DPDK ABI Policy, section 3.5.3.
> https://doc.dpdk.org/guides/contributing/abi_policy.html
>
> Thanks,
>
> The DPDK Symbol Bot
>
> Symbol Contributor
> Email
> rte_rib6_get_ip Stephen Hemminger
> stephen@networkplumber.org
>
>
> --------------------------------------------------------------------------------
>
>
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v10 2/3] devtools: script to send notifications of expired symbols
2021-09-01 12:46 ` Aaron Conole
2021-09-03 11:15 ` Kinsella, Ray
@ 2021-09-03 13:32 ` Kinsella, Ray
1 sibling, 0 replies; 52+ messages in thread
From: Kinsella, Ray @ 2021-09-03 13:32 UTC (permalink / raw)
To: Aaron Conole
Cc: dev, bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor
On 01/09/2021 13:46, Aaron Conole wrote:
> Ray Kinsella <mdr@ashroe.eu> writes:
>
>> Use this script with the output of the DPDK symbol tool, to notify
>> maintainers of expired symbols by email. You need to define the environment
>> variable DPDK_GETMAINTAINER_PATH for this tool to work.
>>
>> Use terminal output to review the emails before sending.
>> e.g.
>> $ devtools/symbol-tool.py list-expired --format-output csv \
>> | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
>> devtools/notify_expired_symbols.py --format-output terminal
>>
>> Then use email output to send the emails to the maintainers.
>> e.g.
>> $ devtools/symbol-tool.py list-expired --format-output csv \
>> | DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
>> devtools/notify_expired_symbols.py --format-output email \
>> --smtp-server <server> --sender <someone@somewhere.com> \
>> --password <password> --cc <someone@somewhere.com>
>>
>> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
>> ---
>> devtools/notify-symbol-maintainers.py | 256 ++++++++++++++++++++++++++
>> 1 file changed, 256 insertions(+)
>> create mode 100755 devtools/notify-symbol-maintainers.py
>>
>> diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py
>> new file mode 100755
>> index 0000000000..ee554687ff
>> --- /dev/null
>> +++ b/devtools/notify-symbol-maintainers.py
>> @@ -0,0 +1,256 @@
>> +#!/usr/bin/env python3
>> +# SPDX-License-Identifier: BSD-3-Clause
>> +# Copyright(c) 2021 Intel Corporation
>> +# pylint: disable=invalid-name
>> +'''Tool to notify maintainers of expired symbols'''
>> +import smtplib
>> +import ssl
>> +import sys
>> +import subprocess
>> +import argparse
>> +from argparse import RawTextHelpFormatter
>> +import time
>> +from email.message import EmailMessage
>> +
>> +DESCRIPTION = '''
>> +Use this script with the output of the DPDK symbol tool, to notify maintainers
>> +and contributors of expired symbols by email. You need to define the environment
>> +variable DPDK_GETMAINTAINER_PATH for this tool to work.
>> +
>> +Use terminal output to review the emails before sending.
>> +e.g.
>> +$ devtools/symbol-tool.py list-expired --format-output csv \\
>> +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
>> +{s} --format-output terminal
>> +
>> +Then use email output to send the emails to the maintainers.
>> +e.g.
>> +$ devtools/symbol-tool.py list-expired --format-output csv \\
>> +| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
>> +{s} --format-output email \\
>> +--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\
>> +--cc <someone@somewhere.com>
>> +'''
>> +
>> +EMAIL_TEMPLATE = '''Hi there,
>> +
>> +Please note the symbols listed below have expired. In line with the DPDK ABI
>> +policy, they should be scheduled for removal, in the next DPDK release.
>> +
>> +For more information, please see the DPDK ABI Policy, section 3.5.3.
>> +https://doc.dpdk.org/guides/contributing/abi_policy.html
>> +
>> +Thanks,
>> +
>> +The DPDK Symbol Bot
>> +
>> +'''
>> +
>> +ABI_POLICY = 'doc/guides/contributing/abi_policy.rst'
>> +MAINTAINERS = 'MAINTAINERS'
>> +get_maintainer = ['devtools/get-maintainer.sh', \
>> + '--email', '-f']
>
> Maybe it's best to make this something that can be overridden. There's
> a series to change the .sh files to .py files. Perhaps an environment
> variable or argument?
>
>> +def _get_maintainers(libpath):
>> + '''Get the maintainers for given library'''
>> + try:
>> + cmd = get_maintainer + [libpath]
>> + result = subprocess.run(cmd, \
>> + stdout=subprocess.PIPE, \
>> + stderr=subprocess.PIPE,
>> + check=True)
>> + except subprocess.CalledProcessError:
>> + return None
>
> You might consider handling
>
> except FileNotFoundError:
> ....
>
> With a graceful exit and error message. In case the get_maintainers
> path changes.
>
So FYI - this get's into the weed a bit.
As there is already a DPDK_GETMAINTAINER_PATH environment variable,
what would you call a new variable.
So instead I added logic for the script to sanity check that _everything_
is defined and where it expects it to be, and then complain loudly
and die when it is not.
The devtools scripts already cross-reference either each, so I'd expect
any changes changing to get-maintainers.sh to get-maintainers.py to take
care of cross-references.
Ray K
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v10 2/3] devtools: script to send notifications of expired symbols
2021-09-01 13:01 ` David Marchand
2021-09-03 13:28 ` Kinsella, Ray
@ 2021-09-03 13:34 ` Kinsella, Ray
1 sibling, 0 replies; 52+ messages in thread
From: Kinsella, Ray @ 2021-09-03 13:34 UTC (permalink / raw)
To: David Marchand
Cc: dev, Bruce Richardson, Stephen Hemminger, Yigit, Ferruh,
Thomas Monjalon, Kevin Traynor, Aaron Conole
On 01/09/2021 14:01, David Marchand wrote:
> Hello Ray,
>
> On Tue, Aug 31, 2021 at 4:51 PM Ray Kinsella <mdr@ashroe.eu> wrote:
>>
>> Use this script with the output of the DPDK symbol tool, to notify
>> maintainers of expired symbols by email. You need to define the environment
>> variable DPDK_GETMAINTAINER_PATH for this tool to work.
>>
>> Use terminal output to review the emails before sending.
>
> Two comments:
> - there are references of a previous name for the script,
> %s/notify_expired_symbols.py/notify-symbol-maintainers.py/g
>
> - and a reminder for the empty report that we received yesterday.
> I think this can be reproduced with:
>
I err'ed - this taken care of v11 also.
Ray K
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v11 0/3] devtools: scripts to count and track symbols
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
` (9 preceding siblings ...)
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 " Ray Kinsella
@ 2021-09-08 15:12 ` Ray Kinsella
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 1/3] devtools: script to track symbols over releases Ray Kinsella
` (2 more replies)
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 0/4] devtools: scripts to count and track symbols Ray Kinsella
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 0/4] devtools: scripts to count and track symbols Ray Kinsella
12 siblings, 3 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-08 15:12 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
Scripts to count and track the lifecycle of DPDK symbols.
The symbol-tool script reports on the growth of symbols over releases
and list expired symbols. The notify-symbol-maintainers script
consumes the input from symbol-tool and generates email notifications
of expired symbols.
v2: reworked to fix pylint errors
v3: sent with the correct in-reply-to
v4: fix typos picked up by the CI
v5: fix terminal_size & directory args
v6: added list-expired, to list expired experimental symbols
v7: fix typo in comments
v8: added tool to notify maintainers of expired symbols
v9: removed hardcoded emails addressed and script names
v10: added ability to identify and notify the original contributors
v11: addressed feedback from Aaron Conole, including PEP8 errors.
Ray Kinsella (3):
devtools: script to track symbols over releases
devtools: script to send notifications of expired symbols
maintainers: add new abi scripts
MAINTAINERS | 2 +
devtools/notify-symbol-maintainers.py | 302 +++++++++++++++
devtools/symbol-tool.py | 505 ++++++++++++++++++++++++++
3 files changed, 809 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
create mode 100755 devtools/symbol-tool.py
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v11 1/3] devtools: script to track symbols over releases
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 0/3] devtools: scripts to count and track symbols Ray Kinsella
@ 2021-09-08 15:12 ` Ray Kinsella
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 2/3] devtools: script to send notifications of expired symbols Ray Kinsella
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 3/3] maintainers: add new abi scripts Ray Kinsella
2 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-08 15:12 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
This script tracks the growth of stable and experimental symbols
over releases since v19.11. The script has the ability to
count the added symbols between two dpdk releases, and to
list experimental symbols present in two dpdk releases
(expired symbols).
example usages:
Count symbols added since v19.11
$ devtools/symbol-tool.py count-symbols
Count symbols added since v20.11
$ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05
List experimental symbols present in v20.11 and v21.05
$ devtools/symbol-tool.py list-expired --releases v20.11,v21.05
List experimental symbols in libraries only, present since v19.11
$ devtools/symbol-tool.py list-expired --directory lib
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/symbol-tool.py | 505 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 505 insertions(+)
create mode 100755 devtools/symbol-tool.py
diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py
new file mode 100755
index 0000000000..a0b81c1b90
--- /dev/null
+++ b/devtools/symbol-tool.py
@@ -0,0 +1,505 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+# pylint: disable=invalid-name
+'''Tool to count or list symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import re
+import datetime
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ sys.exit()
+
+DESCRIPTION = '''
+This script tracks the growth of stable and experimental symbols
+over releases since v19.11. The script has the ability to
+count the added symbols between two dpdk releases, and to
+list experimental symbols present in two dpdk releases
+(expired symbols), including the name & email of the original contributor.
+
+example usages:
+
+Count symbols added since v19.11
+$ {s} count-symbols
+
+Count symbols added since v20.11
+$ {s} count-symbols --releases v20.11,v21.05
+
+List experimental symbols present in v20.11 and v21.05
+$ {s} list-expired --releases v20.11,v21.05
+
+List experimental symbols in libraries only, present since v19.11
+$ {s} list-expired --directory lib
+'''
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+""" # noqa: E501
+
+
+def get_abi_versions():
+ '''Returns a string of possible dpdk abi versions'''
+
+ year = datetime.date.today().year - 2000
+ tags = " |".join(['\'{}\''.format(i)
+ for i in reversed(range(21, year + 1))])
+ tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return tags
+
+
+def get_dpdk_releases():
+ '''Returns a list of dpdk release tags names since v19.11'''
+
+ year = datetime.date.today().year - 2000
+ year_range = "|".join("{}".format(i) for i in range(19, year + 1))
+ pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ try:
+ result = subprocess.run(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ print("Failed to interogate git for release tags")
+ sys.exit()
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [tag.replace('\"', '')
+ for tag in reversed(tags)
+ if pattern.match(tag)][:-3]
+
+ return tags
+
+
+def fix_directory_name(path):
+ '''Prepend librte to the source directory name'''
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+
+def directory_renamed(path, rel):
+ '''Fix removal of the librte_ from the directory names'''
+
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ return result
+
+
+def mapfile_renamed(path, rel):
+ '''Fix renaming of the map file'''
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree',
+ rel, str(path.parent) + '/'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if newfile is not None:
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+
+def mapfile_and_directory_renamed(path, rel):
+ '''Fix renaming of the map file & the source directory'''
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path), path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+
+FIX_STRATEGIES = [directory_renamed,
+ mapfile_renamed,
+ mapfile_and_directory_renamed]
+
+
+def get_symbols(map_parser, release, mapfile_path):
+ '''Count the symbols for a given release and mapfile'''
+ abi_sections = {}
+
+ tagfile = '{}:{}'.format(release, mapfile_path)
+ try:
+ result = subprocess.run(['git', 'show', tagfile],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ for fix_strategy in FIX_STRATEGIES:
+ if result is not None:
+ break
+ result = fix_strategy(mapfile_path, release)
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = map_parser(mapfile).abi()
+
+ return abi_sections
+
+
+def get_terminal_rows():
+ '''Find the number of rows in the terminal'''
+
+ try:
+ return os.get_terminal_size().lines
+ except IOError:
+ return 0
+
+
+class SymbolOwner():
+ '''Find the symbols original contributors name and email'''
+ symbol_regex = {}
+ blame_regex = {'name': r'author\s(.*)',
+ 'email': r'author-mail\s<(.*)>'}
+
+ def __init__(self, libpath, symbol):
+ self.libpath = libpath
+ self.symbol = symbol
+
+ # find variable definitions in C files, and functions in headers.
+ self.symbol_regex = \
+ {'*.c': r'^(?!extern).*' + self.symbol + '[^()]*;',
+ '*.h': r'__rte_experimental(?:.*\n){0,2}.*' + self.symbol}
+
+ def find_symbol_location(self):
+ '''Find where the symbol is definited in the source'''
+ for key in self.symbol_regex:
+ for path in Path(self.libpath).rglob(key):
+ file_text = open(path).read()
+
+ # find where the symbol is defined, either preceeded by
+ # rte_experimental tag (functions)
+ # or followed by a ; (variables)
+
+ exp = self.symbol_regex[key]
+ pattern = re.compile(exp, re.MULTILINE)
+ search = pattern.search(file_text)
+
+ if search is not None:
+ symbol_pos = search.span()[1]
+ symbol_line = file_text.count('\n', 0, symbol_pos) + 1
+
+ return [str(path), symbol_line]
+ return None
+
+ def find_symbol_owner(self):
+ '''Find the symbols original contributors name and email'''
+ owners = {}
+ location = self.find_symbol_location()
+
+ if location is None:
+ return None
+
+ line = '-L {},{}'.format(location[1], location[1])
+ # git blame -p(orcelain) -L(ine) path
+ args = ['-p', line, location[0]]
+
+ try:
+ result = subprocess.run(['git', 'blame'] + args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ return None
+
+ blame = result.stdout.decode('utf-8')
+ for key in self.blame_regex:
+ pattern = re.compile(self.blame_regex[key], re.MULTILINE)
+ match = pattern.search(blame)
+
+ owners[key] = match.groups()[0]
+
+ return owners
+
+
+class SymbolCountOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.OUTPUT_FORMATS[format_output](self, dpdk_releases)
+ self.column_titles = ['mapfile'] + dpdk_releases
+
+ self.terminal_rows = get_terminal_rows()
+ self.row = 0
+
+ def set_terminal_output(self, dpdk_rel):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}' + \
+ ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+ self.column_fmt = '{:50}' + \
+ ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+ def set_csv_output(self, dpdk_rel):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},' + \
+ ','.join(['{},{}'] * (len(dpdk_rel)))
+ self.column_fmt = '{},' + \
+ ','.join(['{},'] * (len(dpdk_rel)))
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+ self.row += 1
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+ mapfile = str(mapfile)
+ print(self.output_fmt.format(*([mapfile] + symbols)))
+ self.row += 1
+
+ if((self.terminal_rows > 0) and
+ ((self.row % self.terminal_rows) == 0)):
+ self.print_columns()
+
+ OUTPUT_FORMATS = {None: set_terminal_output,
+ 'terminal': set_terminal_output,
+ 'csv': set_csv_output}
+
+
+class ListExpiredOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.terminal = True
+ self.OUTPUT_FORMATS[format_output](self, dpdk_releases)
+ self.column_titles = ['mapfile'] + \
+ ['expired (' + ','.join(dpdk_releases) + ')'] + \
+ ['contributor name', 'contributor email']
+
+ def set_terminal_output(self, _):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}{:<50}{:<25}{:<25}'
+ self.column_fmt = '{:50}{:50}{:25}{:25}'
+
+ def set_csv_output(self, _):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},{},{},{}'
+ self.column_fmt = '{},{},{},{}'
+ self.terminal = False
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+
+ def print_row(self, mapfile, symbols, owner):
+ '''Print row of symbol values'''
+
+ for symbol in symbols:
+ mapfile = str(mapfile)
+ name = owner[symbol]['name'] \
+ if owner[symbol] is not None else ''
+ email = owner[symbol]['email'] \
+ if owner[symbol] is not None else ''
+
+ print(self.output_fmt.format(mapfile, symbol, name, email))
+ if self.terminal:
+ mapfile = ''
+
+ OUTPUT_FORMATS = {None: set_terminal_output,
+ 'terminal': set_terminal_output,
+ 'csv': set_csv_output}
+
+
+class CountSymbolsAction:
+ ''' Logic to count symbols added since a give release '''
+ IGNORE_SECTIONS = ['EXPERIMENTAL', 'INTERNAL']
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.symbols_count = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbol_count = experimental_count = 0
+
+ symbols = get_symbols(self.parser, release, self.path)
+
+ # which versions are present, and we care about
+ abi_vers = [abi_ver
+ for abi_ver in symbols
+ if abi_ver not in self.IGNORE_SECTIONS]
+
+ for abi_ver in abi_vers:
+ symbol_count += len(symbols[abi_ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental_count = len(symbols['EXPERIMENTAL'])
+
+ self.symbols_count += [symbol_count, experimental_count]
+
+ def __del__(self):
+ self.format_output.print_row(self.path.parent, self.symbols_count)
+
+
+class ListExpiredAction:
+ ''' Logic to list expired symbols between two releases '''
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.experimental_symbols = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbols = get_symbols(self.parser, release, self.path)
+
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental = [exp.strip() for exp in symbols['EXPERIMENTAL']]
+
+ self.experimental_symbols.append(experimental)
+
+ def __del__(self):
+ if len(self.experimental_symbols) != 2:
+ return
+
+ tmp = self.experimental_symbols
+ # find symbols present in both dpdk releases
+ intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]]
+
+ # check for empty set
+ if intersect_syms == []:
+ return
+
+ sym_owner = {}
+ for sym in intersect_syms:
+ sym_owner[sym] = \
+ SymbolOwner(self.path.parent, sym).find_symbol_owner()
+
+ self.format_output.print_row(self.path.parent,
+ intersect_syms,
+ sym_owner)
+
+
+SRC_DIRECTORIES = 'drivers,lib'
+
+ACTIONS = {None: CountSymbolsAction,
+ 'count-symbols': CountSymbolsAction,
+ 'list-expired': ListExpiredAction}
+
+ACTION_OUTPUT = {None: SymbolCountOutput,
+ 'count-symbols': SymbolCountOutput,
+ 'list-expired': ListExpiredOutput}
+
+
+def main():
+ '''Main entry point'''
+
+ dpdk_releases = get_dpdk_releases()
+
+ parser = \
+ argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__),
+ formatter_class=RawTextHelpFormatter)
+
+ parser.add_argument('mode', choices=['count-symbols', 'list-expired'])
+ parser.add_argument('--format-output', choices=['terminal', 'csv'],
+ default='terminal')
+ parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','),
+ default=SRC_DIRECTORIES)
+ parser.add_argument('--releases',
+ help='2 x comma separated release tags e.g. \''
+ + ','.join([dpdk_releases[0], dpdk_releases[-1]])
+ + '\'')
+ args = parser.parse_args()
+
+ if args.releases is not None:
+ dpdk_releases = args.releases.split(',')
+
+ if args.mode == 'list-expired':
+ if len(dpdk_releases) < 2:
+ sys.exit('Please specify two releases to compare '
+ 'in \'list-expired\' mode.')
+ dpdk_releases = [dpdk_releases[0],
+ dpdk_releases[len(dpdk_releases) - 1]]
+
+ action = ACTIONS[args.mode]
+ format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases)
+
+ map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+ map_parser = makeGrammar(map_grammar, {})
+
+ format_output.print_columns()
+
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ release_action = action(path, map_parser, format_output)
+
+ for release in dpdk_releases:
+ release_action.add_mapfile(release)
+
+ # all the magic happens in the destructor
+ del release_action
+
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v11 2/3] devtools: script to send notifications of expired symbols
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 0/3] devtools: scripts to count and track symbols Ray Kinsella
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 1/3] devtools: script to track symbols over releases Ray Kinsella
@ 2021-09-08 15:12 ` Ray Kinsella
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 3/3] maintainers: add new abi scripts Ray Kinsella
2 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-08 15:12 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
Use this script with the output of the DPDK symbol tool, to notify
maintainers of expired symbols by email. You need to define the environment
variable DPDK_GETMAINTAINER_PATH for this tool to work.
Use terminal output to review the emails before sending.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
devtools/notify_expired_symbols.py --format-output terminal
Then use email output to send the emails to the maintainers.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
devtools/notify_expired_symbols.py --format-output email \
--smtp-server <server> --sender <someone@somewhere.com> \
--password <password> --cc <someone@somewhere.com>
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/notify-symbol-maintainers.py | 302 ++++++++++++++++++++++++++
1 file changed, 302 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py
new file mode 100755
index 0000000000..edf330f88b
--- /dev/null
+++ b/devtools/notify-symbol-maintainers.py
@@ -0,0 +1,302 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+# pylint: disable=invalid-name
+'''Tool to notify maintainers of expired symbols'''
+import os
+import smtplib
+import ssl
+import sys
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import time
+from email.message import EmailMessage
+from pathlib import Path
+
+DESCRIPTION = '''
+Use this script with the output of the DPDK symbol tool, to notify maintainers
+and contributors of expired symbols by email. You need to define the environment
+variable DPDK_GETMAINTAINER_PATH for this tool to work.
+
+Use terminal output to review the emails before sending.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+{s} --format-output terminal
+
+Then use email output to send the emails to the maintainers.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+{s} --format-output email \\
+--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\
+--cc <someone@somewhere.com>
+''' # noqa: E501
+
+EMAIL_TEMPLATE = '''Hi there,
+
+Please note the symbols listed below have expired. In line with the DPDK ABI
+policy, they should be scheduled for removal, in the next DPDK release.
+
+For more information, please see the DPDK ABI Policy, section 3.5.3.
+https://doc.dpdk.org/guides/contributing/abi_policy.html
+
+Thanks,
+
+The DPDK Symbol Bot
+
+''' # noqa: E501
+
+ABI_POLICY = 'doc/guides/contributing/abi_policy.rst'
+DPDK_GMP_ENV_VAR = 'DPDK_GETMAINTAINER_PATH'
+MAINTAINERS = 'MAINTAINERS'
+get_maintainer = ['devtools/get-maintainer.sh',
+ '--email', '-f']
+
+
+class EnvironException(Exception):
+ '''Subclass exception for Pylint\'s happiness.'''
+
+
+def _die_on_exception(e):
+ '''Print an exception, and quit'''
+
+ print('Fatal Error: ' + str(e))
+ sys.exit()
+
+
+def _check_get_maintainers_env():
+ '''Check get maintainers scripts are setup'''
+
+ if not Path(get_maintainer[0]).is_file():
+ raise EnvironException('Cannot locate DPDK\'s get maintainers script, '
+ ' usually at $' + get_maintainer[0] + '.')
+
+ if DPDK_GMP_ENV_VAR not in os.environ:
+ raise EnvironException(DPDK_GMP_ENV_VAR + ' is not defined.')
+
+ if not Path(os.environ[DPDK_GMP_ENV_VAR]).is_file():
+ raise EnvironException('Cannot locate get maintainers script, usually'
+ ' at ' + DPDK_GMP_ENV_VAR + '.')
+
+
+def _get_maintainers(libpath):
+ '''Get the maintainers for given library'''
+
+ try:
+ _check_get_maintainers_env()
+ except EnvironException as e:
+ _die_on_exception(e)
+
+ try:
+ cmd = get_maintainer + [libpath]
+ result = subprocess.run(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError as e:
+ _die_on_exception(e)
+
+ if result is None:
+ return None
+
+ email = result.stdout.decode('utf-8')
+ if email == '':
+ return None
+
+ email = list(filter(None, email.split('\n')))
+ return email
+
+
+default_maintainers = _get_maintainers(ABI_POLICY) + \
+ _get_maintainers(MAINTAINERS)
+
+
+def get_maintainers(libpath):
+ '''Get the maintainers for given library'''
+ maintainers = _get_maintainers(libpath)
+
+ if maintainers is None:
+ maintainers = default_maintainers
+
+ return maintainers
+
+
+def get_message(library, symbols, config):
+ '''Build email message from symbols, config and maintainers'''
+ contributors = {}
+ message = {}
+ maintainers = get_maintainers(library)
+
+ if maintainers != default_maintainers:
+ message['CC'] = default_maintainers.copy()
+
+ if 'CC' in config:
+ message.setdefault('CC', []).append(config['CC'])
+
+ message['Subject'] = 'Expired symbols in {}\n'.format(library)
+
+ body = EMAIL_TEMPLATE
+ body += '{:<50}{:<25}{:<25}\n'.format('Symbol', 'Contributor', 'Email')
+ for sym in symbols:
+ body += ('{:<50}{:<25}{:<25}\n'.format(sym,
+ symbols[sym]['name'],
+ symbols[sym]['email']))
+ email = symbols[sym]['email']
+ contributors[email] = ''
+
+ contributors = list(contributors.keys())
+
+ message['To'] = maintainers + contributors
+ message['Body'] = body
+
+ return message
+
+
+class OutputEmail():
+ '''Format the output for email'''
+
+ def __init__(self, config):
+ self.config = config
+
+ self.terminal = OutputTerminal(config)
+ context = ssl.create_default_context()
+
+ # Try to log in to server and send email
+ try:
+ self.server = smtplib.SMTP(config['smtp_server'], 587)
+ self.server.starttls(context=context) # Secure the connection
+ self.server.login(config['sender'], config['password'])
+ except EnvironException as e:
+ _die_on_exception(e)
+
+ def message(self, message):
+ '''send email'''
+ self.terminal.message(message)
+
+ msg = EmailMessage()
+ msg.set_content(message.pop('Body'))
+
+ for key in message.keys():
+ msg[key] = message[key]
+
+ msg['From'] = self.config['sender']
+ msg['Reply-To'] = 'no-reply@dpdk.org'
+
+ self.server.send_message(msg)
+
+ time.sleep(1)
+
+ def __del__(self):
+ self.server.quit()
+
+
+class OutputTerminal(): # pylint: disable=too-few-public-methods
+ '''Format the output for the terminal'''
+
+ def __init__(self, config):
+ self.config = config
+
+ def message(self, message):
+ '''Print email to terminal'''
+
+ terminal = 'To:' + ', '.join(message['To']) + '\n'
+ if 'sender' in self.config.keys():
+ terminal += 'From:' + self.config['sender'] + '\n'
+
+ terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n'
+
+ if 'CC' in message:
+ terminal += 'CC:' + ', '.join(message['CC']) + '\n'
+
+ terminal += 'Subject:' + message['Subject'] + '\n'
+ terminal += 'Body:' + message['Body'] + '\n'
+
+ print(terminal)
+ print('-' * 80)
+
+
+def parse_config(args):
+ '''put the command line args in the right places'''
+ config = {}
+ error_msg = None
+
+ outputs = {
+ None: OutputTerminal,
+ 'terminal': OutputTerminal,
+ 'email': OutputEmail
+ }
+
+ if args.format_output == 'email':
+ if args.smtp_server is None:
+ error_msg = 'SMTP server'
+ else:
+ config['smtp_server'] = args.smtp_server
+
+ if args.sender is None:
+ error_msg = 'sender'
+ else:
+ config['sender'] = args.sender
+
+ if args.password is None:
+ error_msg = 'password'
+ else:
+ config['password'] = args.password
+
+ if args.cc is not None:
+ config['CC'] = args.cc
+
+ if error_msg is not None:
+ print('Please specify a {} for email output'.format(error_msg))
+ return None
+
+ config['output'] = outputs[args.format_output]
+ return config
+
+
+def main():
+ '''Main entry point'''
+ parser = \
+ argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__),
+ formatter_class=RawTextHelpFormatter)
+ parser.add_argument('--format-output',
+ choices=['terminal', 'email'],
+ default='terminal')
+ parser.add_argument('--smtp-server')
+ parser.add_argument('--password')
+ parser.add_argument('--sender')
+ parser.add_argument('--cc')
+
+ args = parser.parse_args()
+ config = parse_config(args)
+ if config is None:
+ return
+
+ symbols = {}
+ lastlib = library = ''
+
+ output = config['output'](config)
+
+ for line in sys.stdin:
+ line = line.rstrip('\n')
+
+ if line.find('mapfile') >= 0:
+ continue
+ library, symbol, name, email = line.split(',')
+
+ if library != lastlib:
+ message = get_message(lastlib, symbols, config)
+ output.message(message)
+ symbols = {}
+
+ lastlib = library
+ symbols[symbol] = {'name': name, 'email': email}
+
+ # print the last library
+ message = get_message(lastlib, symbols, config)
+ output.message(message)
+
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v11 3/3] maintainers: add new abi scripts
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 0/3] devtools: scripts to count and track symbols Ray Kinsella
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 1/3] devtools: script to track symbols over releases Ray Kinsella
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 2/3] devtools: script to send notifications of expired symbols Ray Kinsella
@ 2021-09-08 15:12 ` Ray Kinsella
2 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-08 15:12 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
Add new abi management scripts to the MAINTAINERS file.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
MAINTAINERS | 2 ++
1 file changed, 2 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 266f5ac1da..ff8245271f 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -129,6 +129,8 @@ F: devtools/gen-abi.sh
F: devtools/libabigail.abignore
F: devtools/update-abi.sh
F: devtools/update_version_map_abi.py
+F: devtools/notify-symbol-maintainers.py
+F: devtools/symbol-tool.py
F: buildtools/check-symbols.sh
F: buildtools/map-list-symbol.sh
F: drivers/*/*/*.map
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v12 0/4] devtools: scripts to count and track symbols
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
` (10 preceding siblings ...)
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 0/3] devtools: scripts to count and track symbols Ray Kinsella
@ 2021-09-08 15:13 ` Ray Kinsella
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 1/4] devtools: script to track symbols over releases Ray Kinsella
` (3 more replies)
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 0/4] devtools: scripts to count and track symbols Ray Kinsella
12 siblings, 4 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-08 15:13 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
Scripts to count and track the lifecycle of DPDK symbols.
The symbol-tool script reports on the growth of symbols over releases
and list expired symbols. The notify-symbol-maintainers script
consumes the input from symbol-tool and generates email notifications
of expired symbols.
v2: reworked to fix pylint errors
v3: sent with the correct in-reply-to
v4: fix typos picked up by the CI
v5: fix terminal_size & directory args
v6: added list-expired, to list expired experimental symbols
v7: fix typo in comments
v8: added tool to notify maintainers of expired symbols
v9: removed hardcoded emails addressed and script names
v10: added ability to identify and notify the original contributors
v11: addressed feedback from Aaron Conole, including PEP8 errors.
v12: added symbol-tool ignore functionality, to ignore specific symbols.
Ray Kinsella (4):
devtools: script to track symbols over releases
devtools: script to send notifications of expired symbols
maintainers: add new abi scripts
devtools: add asym crypto to symbol-tool ignore
MAINTAINERS | 2 +
devtools/notify-symbol-maintainers.py | 302 ++++++++++++++
devtools/symbol-tool.py | 566 ++++++++++++++++++++++++++
devtools/symboltool.ignore | 3 +
4 files changed, 873 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
create mode 100755 devtools/symbol-tool.py
create mode 100644 devtools/symboltool.ignore
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v12 1/4] devtools: script to track symbols over releases
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 0/4] devtools: scripts to count and track symbols Ray Kinsella
@ 2021-09-08 15:13 ` Ray Kinsella
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 2/4] devtools: script to send notifications of expired symbols Ray Kinsella
` (2 subsequent siblings)
3 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-08 15:13 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
This script tracks the growth of stable and experimental symbols
over releases since v19.11. The script has the ability to
count the added symbols between two dpdk releases, and to
list experimental symbols present in two dpdk releases
(expired symbols).
example usages:
Count symbols added since v19.11
$ devtools/symbol-tool.py count-symbols
Count symbols added since v20.11
$ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05
List experimental symbols present in v20.11 and v21.05
$ devtools/symbol-tool.py list-expired --releases v20.11,v21.05
List experimental symbols in libraries only, present since v19.11
$ devtools/symbol-tool.py list-expired --directory lib
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/symbol-tool.py | 566 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 566 insertions(+)
create mode 100755 devtools/symbol-tool.py
diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py
new file mode 100755
index 0000000000..a71ab59539
--- /dev/null
+++ b/devtools/symbol-tool.py
@@ -0,0 +1,566 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+# pylint: disable=invalid-name
+'''Tool to count or list symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import re
+import datetime
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ sys.exit()
+
+DESCRIPTION = '''
+This script tracks the growth of stable and experimental symbols
+over releases since v19.11. The script has the ability to
+count the added symbols between two dpdk releases, and to
+list experimental symbols present in two dpdk releases
+(expired symbols), including the name & email of the original contributor.
+
+example usages:
+
+Count symbols added since v19.11
+$ {s} count-symbols
+
+Count symbols added since v20.11
+$ {s} count-symbols --releases v20.11,v21.05
+
+List experimental symbols present in v20.11 and v21.05
+$ {s} list-expired --releases v20.11,v21.05
+
+List experimental symbols in libraries only, present since v19.11
+$ {s} list-expired --directory lib
+'''
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+""" # noqa: E501
+
+
+class EnvironException(Exception):
+ '''Subclass exception for Pylint\'s happiness.'''
+
+
+def get_abi_versions():
+ '''Returns a string of possible dpdk abi versions'''
+
+ year = datetime.date.today().year - 2000
+ tags = " |".join(['\'{}\''.format(i)
+ for i in reversed(range(21, year + 1))])
+ tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return tags
+
+
+def get_dpdk_releases():
+ '''Returns a list of dpdk release tags names since v19.11'''
+
+ year = datetime.date.today().year - 2000
+ year_range = "|".join("{}".format(i) for i in range(19, year + 1))
+ pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ try:
+ result = subprocess.run(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ print("Failed to interogate git for release tags")
+ sys.exit()
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [tag.replace('\"', '')
+ for tag in reversed(tags)
+ if pattern.match(tag)][:-3]
+
+ return tags
+
+
+def fix_directory_name(path):
+ '''Prepend librte to the source directory name'''
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+
+def directory_renamed(path, rel):
+ '''Fix removal of the librte_ from the directory names'''
+
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ return result
+
+
+def mapfile_renamed(path, rel):
+ '''Fix renaming of the map file'''
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree',
+ rel, str(path.parent) + '/'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if newfile is not None:
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+
+def mapfile_and_directory_renamed(path, rel):
+ '''Fix renaming of the map file & the source directory'''
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path), path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+
+FIX_STRATEGIES = [directory_renamed,
+ mapfile_renamed,
+ mapfile_and_directory_renamed]
+
+
+def get_symbols(map_parser, release, mapfile_path):
+ '''Count the symbols for a given release and mapfile'''
+ abi_sections = {}
+
+ tagfile = '{}:{}'.format(release, mapfile_path)
+ try:
+ result = subprocess.run(['git', 'show', tagfile],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ for fix_strategy in FIX_STRATEGIES:
+ if result is not None:
+ break
+ result = fix_strategy(mapfile_path, release)
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = map_parser(mapfile).abi()
+
+ return abi_sections
+
+
+def get_terminal_rows():
+ '''Find the number of rows in the terminal'''
+
+ try:
+ return os.get_terminal_size().lines
+ except IOError:
+ return 0
+
+
+class IgnoredSymbols(): # pylint: disable=too-few-public-methods
+ '''Symbols which are to be be ignored for some period'''
+
+ SYMBOL_TOOL_IGNORE = 'devtools/symboltool.ignore'
+ ignore_regex = []
+ __initialized = False
+
+ @staticmethod
+ def initialize():
+ '''intialize once'''
+
+ if IgnoredSymbols.__initialized:
+ return
+ IgnoredSymbols.__initialized = True
+
+ if 'DPDK_SYMBOL_TOOL_IGNORE' in os.environ:
+ IgnoredSymbols.SYMBOL_TOOL_IGNORE = \
+ os.environ['DPDK_SYMBOL_TOOL_IGNORE']
+
+ # if the user specifies an ignore file, we can't find then error.
+ if not Path(IgnoredSymbols.SYMBOL_TOOL_IGNORE).is_file():
+ raise EnvironException('Cannot locate {}\'s '
+ 'ignore file'.format(__file__))
+
+ # if we cannot find the default ignore file, then continue
+ if not Path(IgnoredSymbols.SYMBOL_TOOL_IGNORE).is_file():
+ return
+
+ lines = open(Path(IgnoredSymbols.SYMBOL_TOOL_IGNORE)).readlines()
+ for line in lines:
+
+ line = line.strip()
+
+ # ignore comments and whitespace
+ if line.startswith(';') or len(line) == 0:
+ continue
+
+ IgnoredSymbols.ignore_regex.append(re.compile(line))
+
+ def __init__(self):
+ self.initialize()
+
+ def check_ignore(self, symbol):
+ '''Check symbol against the ignore regexes'''
+
+ for exp in self.ignore_regex:
+ if exp.search(symbol) is not None:
+ return True
+
+ return False
+
+
+class SymbolOwner():
+ '''Find the symbols original contributors name and email'''
+ symbol_regex = {}
+ blame_regex = {'name': r'author\s(.*)',
+ 'email': r'author-mail\s<(.*)>'}
+
+ def __init__(self, libpath, symbol):
+ self.libpath = libpath
+ self.symbol = symbol
+
+ # find variable definitions in C files, and functions in headers.
+ self.symbol_regex = \
+ {'*.c': r'^(?!extern).*' + self.symbol + '[^()]*;',
+ '*.h': r'__rte_experimental(?:.*\n){0,2}.*' + self.symbol}
+
+ def find_symbol_location(self):
+ '''Find where the symbol is definited in the source'''
+ for key in self.symbol_regex:
+ for path in Path(self.libpath).rglob(key):
+ file_text = open(path).read()
+
+ # find where the symbol is defined, either preceded by
+ # rte_experimental tag (functions)
+ # or followed by a ; (variables)
+
+ exp = self.symbol_regex[key]
+ pattern = re.compile(exp, re.MULTILINE)
+ search = pattern.search(file_text)
+
+ if search is not None:
+ symbol_pos = search.span()[1]
+ symbol_line = file_text.count('\n', 0, symbol_pos) + 1
+
+ return [str(path), symbol_line]
+ return None
+
+ def find_symbol_owner(self):
+ '''Find the symbols original contributors name and email'''
+ owners = {}
+ location = self.find_symbol_location()
+
+ if location is None:
+ return None
+
+ line = '-L {},{}'.format(location[1], location[1])
+ # git blame -p(orcelain) -L(ine) path
+ args = ['-p', line, location[0]]
+
+ try:
+ result = subprocess.run(['git', 'blame'] + args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ return None
+
+ blame = result.stdout.decode('utf-8')
+ for key in self.blame_regex:
+ pattern = re.compile(self.blame_regex[key], re.MULTILINE)
+ match = pattern.search(blame)
+
+ owners[key] = match.groups()[0]
+
+ return owners
+
+
+class SymbolCountOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.OUTPUT_FORMATS[format_output](self, dpdk_releases)
+ self.column_titles = ['mapfile'] + dpdk_releases
+
+ self.terminal_rows = get_terminal_rows()
+ self.row = 0
+
+ def set_terminal_output(self, dpdk_rel):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}' + \
+ ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+ self.column_fmt = '{:50}' + \
+ ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+ def set_csv_output(self, dpdk_rel):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},' + \
+ ','.join(['{},{}'] * (len(dpdk_rel)))
+ self.column_fmt = '{},' + \
+ ','.join(['{},'] * (len(dpdk_rel)))
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+ self.row += 1
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+ mapfile = str(mapfile)
+ print(self.output_fmt.format(*([mapfile] + symbols)))
+ self.row += 1
+
+ if((self.terminal_rows > 0) and
+ ((self.row % self.terminal_rows) == 0)):
+ self.print_columns()
+
+ OUTPUT_FORMATS = {None: set_terminal_output,
+ 'terminal': set_terminal_output,
+ 'csv': set_csv_output}
+
+
+class ListExpiredOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.terminal = True
+ self.OUTPUT_FORMATS[format_output](self, dpdk_releases)
+ self.column_titles = ['mapfile'] + \
+ ['expired (' + ','.join(dpdk_releases) + ')'] + \
+ ['contributor name', 'contributor email']
+
+ def set_terminal_output(self, _):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}{:<50}{:<25}{:<25}'
+ self.column_fmt = '{:50}{:50}{:25}{:25}'
+
+ def set_csv_output(self, _):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},{},{},{}'
+ self.column_fmt = '{},{},{},{}'
+ self.terminal = False
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+
+ def print_row(self, mapfile, symbols, owner):
+ '''Print row of symbol values'''
+
+ for symbol in symbols:
+ mapfile = str(mapfile)
+ name = owner[symbol]['name'] \
+ if owner[symbol] is not None else ''
+ email = owner[symbol]['email'] \
+ if owner[symbol] is not None else ''
+
+ print(self.output_fmt.format(mapfile, symbol, name, email))
+ if self.terminal:
+ mapfile = ''
+
+ OUTPUT_FORMATS = {None: set_terminal_output,
+ 'terminal': set_terminal_output,
+ 'csv': set_csv_output}
+
+
+class CountSymbolsAction:
+ ''' Logic to count symbols added since a give release '''
+ IGNORE_SECTIONS = ['EXPERIMENTAL', 'INTERNAL']
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.symbols_count = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbol_count = experimental_count = 0
+
+ symbols = get_symbols(self.parser, release, self.path)
+
+ # which versions are present, and we care about
+ abi_vers = [abi_ver
+ for abi_ver in symbols
+ if abi_ver not in self.IGNORE_SECTIONS]
+
+ for abi_ver in abi_vers:
+ symbol_count += len(symbols[abi_ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental_count = len(symbols['EXPERIMENTAL'])
+
+ self.symbols_count += [symbol_count, experimental_count]
+
+ def __del__(self):
+ self.format_output.print_row(self.path.parent, self.symbols_count)
+
+
+class ListExpiredAction:
+ ''' Logic to list expired symbols between two releases '''
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.experimental_symbols = []
+ self.ignored_symbols = IgnoredSymbols()
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbols = get_symbols(self.parser, release, self.path)
+
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental = [exp.strip() for exp in symbols['EXPERIMENTAL']]
+
+ self.experimental_symbols.append(experimental)
+
+ def __del__(self):
+ if len(self.experimental_symbols) != 2:
+ return
+
+ tmp = self.experimental_symbols
+ # find symbols present in both dpdk releases
+ intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]]
+
+ # remove ignored symbols
+ intersect_syms = [sym for sym in intersect_syms if not
+ self.ignored_symbols.check_ignore(sym)]
+
+ # check for empty set
+ if intersect_syms == []:
+ return
+
+ sym_owner = {}
+ for sym in intersect_syms:
+ sym_owner[sym] = \
+ SymbolOwner(self.path.parent, sym).find_symbol_owner()
+
+ self.format_output.print_row(self.path.parent,
+ intersect_syms,
+ sym_owner)
+
+
+SRC_DIRECTORIES = 'drivers,lib'
+
+ACTIONS = {None: CountSymbolsAction,
+ 'count-symbols': CountSymbolsAction,
+ 'list-expired': ListExpiredAction}
+
+ACTION_OUTPUT = {None: SymbolCountOutput,
+ 'count-symbols': SymbolCountOutput,
+ 'list-expired': ListExpiredOutput}
+
+
+def main():
+ '''Main entry point'''
+
+ dpdk_releases = get_dpdk_releases()
+
+ parser = \
+ argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__),
+ formatter_class=RawTextHelpFormatter)
+
+ parser.add_argument('mode', choices=['count-symbols', 'list-expired'])
+ parser.add_argument('--format-output', choices=['terminal', 'csv'],
+ default='terminal')
+ parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','),
+ default=SRC_DIRECTORIES)
+ parser.add_argument('--releases',
+ help='2 x comma separated release tags e.g. \''
+ + ','.join([dpdk_releases[0], dpdk_releases[-1]])
+ + '\'')
+ args = parser.parse_args()
+
+ if args.releases is not None:
+ dpdk_releases = args.releases.split(',')
+
+ if args.mode == 'list-expired':
+ if len(dpdk_releases) < 2:
+ sys.exit('Please specify two releases to compare '
+ 'in \'list-expired\' mode.')
+ dpdk_releases = [dpdk_releases[0],
+ dpdk_releases[len(dpdk_releases) - 1]]
+
+ action = ACTIONS[args.mode]
+ format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases)
+
+ map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+ map_parser = makeGrammar(map_grammar, {})
+
+ format_output.print_columns()
+
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ release_action = action(path, map_parser, format_output)
+
+ for release in dpdk_releases:
+ release_action.add_mapfile(release)
+
+ # all the magic happens in the destructor
+ del release_action
+
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v12 2/4] devtools: script to send notifications of expired symbols
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 0/4] devtools: scripts to count and track symbols Ray Kinsella
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 1/4] devtools: script to track symbols over releases Ray Kinsella
@ 2021-09-08 15:13 ` Ray Kinsella
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 3/4] maintainers: add new abi scripts Ray Kinsella
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 4/4] devtools: add asym crypto to symbol-tool ignore Ray Kinsella
3 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-08 15:13 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
Use this script with the output of the DPDK symbol tool, to notify
maintainers of expired symbols by email. You need to define the environment
variable DPDK_GETMAINTAINER_PATH for this tool to work.
Use terminal output to review the emails before sending.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
devtools/notify_expired_symbols.py --format-output terminal
Then use email output to send the emails to the maintainers.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
devtools/notify_expired_symbols.py --format-output email \
--smtp-server <server> --sender <someone@somewhere.com> \
--password <password> --cc <someone@somewhere.com>
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/notify-symbol-maintainers.py | 302 ++++++++++++++++++++++++++
1 file changed, 302 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py
new file mode 100755
index 0000000000..edf330f88b
--- /dev/null
+++ b/devtools/notify-symbol-maintainers.py
@@ -0,0 +1,302 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+# pylint: disable=invalid-name
+'''Tool to notify maintainers of expired symbols'''
+import os
+import smtplib
+import ssl
+import sys
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import time
+from email.message import EmailMessage
+from pathlib import Path
+
+DESCRIPTION = '''
+Use this script with the output of the DPDK symbol tool, to notify maintainers
+and contributors of expired symbols by email. You need to define the environment
+variable DPDK_GETMAINTAINER_PATH for this tool to work.
+
+Use terminal output to review the emails before sending.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+{s} --format-output terminal
+
+Then use email output to send the emails to the maintainers.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+{s} --format-output email \\
+--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\
+--cc <someone@somewhere.com>
+''' # noqa: E501
+
+EMAIL_TEMPLATE = '''Hi there,
+
+Please note the symbols listed below have expired. In line with the DPDK ABI
+policy, they should be scheduled for removal, in the next DPDK release.
+
+For more information, please see the DPDK ABI Policy, section 3.5.3.
+https://doc.dpdk.org/guides/contributing/abi_policy.html
+
+Thanks,
+
+The DPDK Symbol Bot
+
+''' # noqa: E501
+
+ABI_POLICY = 'doc/guides/contributing/abi_policy.rst'
+DPDK_GMP_ENV_VAR = 'DPDK_GETMAINTAINER_PATH'
+MAINTAINERS = 'MAINTAINERS'
+get_maintainer = ['devtools/get-maintainer.sh',
+ '--email', '-f']
+
+
+class EnvironException(Exception):
+ '''Subclass exception for Pylint\'s happiness.'''
+
+
+def _die_on_exception(e):
+ '''Print an exception, and quit'''
+
+ print('Fatal Error: ' + str(e))
+ sys.exit()
+
+
+def _check_get_maintainers_env():
+ '''Check get maintainers scripts are setup'''
+
+ if not Path(get_maintainer[0]).is_file():
+ raise EnvironException('Cannot locate DPDK\'s get maintainers script, '
+ ' usually at $' + get_maintainer[0] + '.')
+
+ if DPDK_GMP_ENV_VAR not in os.environ:
+ raise EnvironException(DPDK_GMP_ENV_VAR + ' is not defined.')
+
+ if not Path(os.environ[DPDK_GMP_ENV_VAR]).is_file():
+ raise EnvironException('Cannot locate get maintainers script, usually'
+ ' at ' + DPDK_GMP_ENV_VAR + '.')
+
+
+def _get_maintainers(libpath):
+ '''Get the maintainers for given library'''
+
+ try:
+ _check_get_maintainers_env()
+ except EnvironException as e:
+ _die_on_exception(e)
+
+ try:
+ cmd = get_maintainer + [libpath]
+ result = subprocess.run(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError as e:
+ _die_on_exception(e)
+
+ if result is None:
+ return None
+
+ email = result.stdout.decode('utf-8')
+ if email == '':
+ return None
+
+ email = list(filter(None, email.split('\n')))
+ return email
+
+
+default_maintainers = _get_maintainers(ABI_POLICY) + \
+ _get_maintainers(MAINTAINERS)
+
+
+def get_maintainers(libpath):
+ '''Get the maintainers for given library'''
+ maintainers = _get_maintainers(libpath)
+
+ if maintainers is None:
+ maintainers = default_maintainers
+
+ return maintainers
+
+
+def get_message(library, symbols, config):
+ '''Build email message from symbols, config and maintainers'''
+ contributors = {}
+ message = {}
+ maintainers = get_maintainers(library)
+
+ if maintainers != default_maintainers:
+ message['CC'] = default_maintainers.copy()
+
+ if 'CC' in config:
+ message.setdefault('CC', []).append(config['CC'])
+
+ message['Subject'] = 'Expired symbols in {}\n'.format(library)
+
+ body = EMAIL_TEMPLATE
+ body += '{:<50}{:<25}{:<25}\n'.format('Symbol', 'Contributor', 'Email')
+ for sym in symbols:
+ body += ('{:<50}{:<25}{:<25}\n'.format(sym,
+ symbols[sym]['name'],
+ symbols[sym]['email']))
+ email = symbols[sym]['email']
+ contributors[email] = ''
+
+ contributors = list(contributors.keys())
+
+ message['To'] = maintainers + contributors
+ message['Body'] = body
+
+ return message
+
+
+class OutputEmail():
+ '''Format the output for email'''
+
+ def __init__(self, config):
+ self.config = config
+
+ self.terminal = OutputTerminal(config)
+ context = ssl.create_default_context()
+
+ # Try to log in to server and send email
+ try:
+ self.server = smtplib.SMTP(config['smtp_server'], 587)
+ self.server.starttls(context=context) # Secure the connection
+ self.server.login(config['sender'], config['password'])
+ except EnvironException as e:
+ _die_on_exception(e)
+
+ def message(self, message):
+ '''send email'''
+ self.terminal.message(message)
+
+ msg = EmailMessage()
+ msg.set_content(message.pop('Body'))
+
+ for key in message.keys():
+ msg[key] = message[key]
+
+ msg['From'] = self.config['sender']
+ msg['Reply-To'] = 'no-reply@dpdk.org'
+
+ self.server.send_message(msg)
+
+ time.sleep(1)
+
+ def __del__(self):
+ self.server.quit()
+
+
+class OutputTerminal(): # pylint: disable=too-few-public-methods
+ '''Format the output for the terminal'''
+
+ def __init__(self, config):
+ self.config = config
+
+ def message(self, message):
+ '''Print email to terminal'''
+
+ terminal = 'To:' + ', '.join(message['To']) + '\n'
+ if 'sender' in self.config.keys():
+ terminal += 'From:' + self.config['sender'] + '\n'
+
+ terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n'
+
+ if 'CC' in message:
+ terminal += 'CC:' + ', '.join(message['CC']) + '\n'
+
+ terminal += 'Subject:' + message['Subject'] + '\n'
+ terminal += 'Body:' + message['Body'] + '\n'
+
+ print(terminal)
+ print('-' * 80)
+
+
+def parse_config(args):
+ '''put the command line args in the right places'''
+ config = {}
+ error_msg = None
+
+ outputs = {
+ None: OutputTerminal,
+ 'terminal': OutputTerminal,
+ 'email': OutputEmail
+ }
+
+ if args.format_output == 'email':
+ if args.smtp_server is None:
+ error_msg = 'SMTP server'
+ else:
+ config['smtp_server'] = args.smtp_server
+
+ if args.sender is None:
+ error_msg = 'sender'
+ else:
+ config['sender'] = args.sender
+
+ if args.password is None:
+ error_msg = 'password'
+ else:
+ config['password'] = args.password
+
+ if args.cc is not None:
+ config['CC'] = args.cc
+
+ if error_msg is not None:
+ print('Please specify a {} for email output'.format(error_msg))
+ return None
+
+ config['output'] = outputs[args.format_output]
+ return config
+
+
+def main():
+ '''Main entry point'''
+ parser = \
+ argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__),
+ formatter_class=RawTextHelpFormatter)
+ parser.add_argument('--format-output',
+ choices=['terminal', 'email'],
+ default='terminal')
+ parser.add_argument('--smtp-server')
+ parser.add_argument('--password')
+ parser.add_argument('--sender')
+ parser.add_argument('--cc')
+
+ args = parser.parse_args()
+ config = parse_config(args)
+ if config is None:
+ return
+
+ symbols = {}
+ lastlib = library = ''
+
+ output = config['output'](config)
+
+ for line in sys.stdin:
+ line = line.rstrip('\n')
+
+ if line.find('mapfile') >= 0:
+ continue
+ library, symbol, name, email = line.split(',')
+
+ if library != lastlib:
+ message = get_message(lastlib, symbols, config)
+ output.message(message)
+ symbols = {}
+
+ lastlib = library
+ symbols[symbol] = {'name': name, 'email': email}
+
+ # print the last library
+ message = get_message(lastlib, symbols, config)
+ output.message(message)
+
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v12 3/4] maintainers: add new abi scripts
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 0/4] devtools: scripts to count and track symbols Ray Kinsella
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 1/4] devtools: script to track symbols over releases Ray Kinsella
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 2/4] devtools: script to send notifications of expired symbols Ray Kinsella
@ 2021-09-08 15:13 ` Ray Kinsella
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 4/4] devtools: add asym crypto to symbol-tool ignore Ray Kinsella
3 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-08 15:13 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
Add new abi management scripts to the MAINTAINERS file.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
MAINTAINERS | 2 ++
1 file changed, 2 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 266f5ac1da..ff8245271f 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -129,6 +129,8 @@ F: devtools/gen-abi.sh
F: devtools/libabigail.abignore
F: devtools/update-abi.sh
F: devtools/update_version_map_abi.py
+F: devtools/notify-symbol-maintainers.py
+F: devtools/symbol-tool.py
F: buildtools/check-symbols.sh
F: buildtools/map-list-symbol.sh
F: drivers/*/*/*.map
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v12 4/4] devtools: add asym crypto to symbol-tool ignore
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 0/4] devtools: scripts to count and track symbols Ray Kinsella
` (2 preceding siblings ...)
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 3/4] maintainers: add new abi scripts Ray Kinsella
@ 2021-09-08 15:13 ` Ray Kinsella
2021-09-08 15:23 ` [dpdk-dev] [EXT] " Akhil Goyal
3 siblings, 1 reply; 52+ messages in thread
From: Ray Kinsella @ 2021-09-08 15:13 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
Add asym crypto to the symbol-tool's ignore file.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/symboltool.ignore | 3 +++
1 file changed, 3 insertions(+)
create mode 100644 devtools/symboltool.ignore
diff --git a/devtools/symboltool.ignore b/devtools/symboltool.ignore
new file mode 100644
index 0000000000..800c500a82
--- /dev/null
+++ b/devtools/symboltool.ignore
@@ -0,0 +1,3 @@
+; regex of symbols for the symbol-tool to ignore
+
+rte_cryptodev_asym_.*
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [EXT] [PATCH v12 4/4] devtools: add asym crypto to symbol-tool ignore
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 4/4] devtools: add asym crypto to symbol-tool ignore Ray Kinsella
@ 2021-09-08 15:23 ` Akhil Goyal
0 siblings, 0 replies; 52+ messages in thread
From: Akhil Goyal @ 2021-09-08 15:23 UTC (permalink / raw)
To: Ray Kinsella, dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor,
aconole, roy.fan.zhang, arkadiuszx.kusztal
> ----------------------------------------------------------------------
> Add asym crypto to the symbol-tool's ignore file.
>
> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
> ---
Acked-by: Akhil Goyal <gakhil@marvell.com>
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v13 0/4] devtools: scripts to count and track symbols
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
` (11 preceding siblings ...)
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 0/4] devtools: scripts to count and track symbols Ray Kinsella
@ 2021-09-09 13:48 ` Ray Kinsella
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 1/4] devtools: script to track symbols over releases Ray Kinsella
` (4 more replies)
12 siblings, 5 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-09 13:48 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
Scripts to count and track the lifecycle of DPDK symbols.
The symbol-tool script reports on the growth of symbols over releases
and list expired symbols. The notify-symbol-maintainers script
consumes the input from symbol-tool and generates email notifications
of expired symbols.
v2: reworked to fix pylint errors
v3: sent with the correct in-reply-to
v4: fix typos picked up by the CI
v5: fix terminal_size & directory args
v6: added list-expired, to list expired experimental symbols
v7: fix typo in comments
v8: added tool to notify maintainers of expired symbols
v9: removed hardcoded emails addressed and script names
v10: added ability to identify and notify the original contributors
v11: addressed feedback from Aaron Conole, including PEP8 errors.
v12: added symbol-tool ignore functionality, to ignore specific symbols
v13: renamed symboltool.abignore, typos, added ack from Akhil Goyal
Ray Kinsella (4):
devtools: script to track symbols over releases
devtools: script to send notifications of expired symbols
maintainers: add new abi scripts
devtools: add asym crypto to symbol-tool ignore
MAINTAINERS | 3 +
devtools/notify-symbol-maintainers.py | 302 ++++++++++++++
devtools/symbol-tool.py | 566 ++++++++++++++++++++++++++
devtools/symboltool.abignore | 3 +
4 files changed, 874 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
create mode 100755 devtools/symbol-tool.py
create mode 100644 devtools/symboltool.abignore
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v13 1/4] devtools: script to track symbols over releases
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 0/4] devtools: scripts to count and track symbols Ray Kinsella
@ 2021-09-09 13:48 ` Ray Kinsella
2024-08-15 16:39 ` Stephen Hemminger
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 2/4] devtools: script to send notifications of expired symbols Ray Kinsella
` (3 subsequent siblings)
4 siblings, 1 reply; 52+ messages in thread
From: Ray Kinsella @ 2021-09-09 13:48 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
This script tracks the growth of stable and experimental symbols
over releases since v19.11. The script has the ability to
count the added symbols between two dpdk releases, and to
list experimental symbols present in two dpdk releases
(expired symbols).
example usages:
Count symbols added since v19.11
$ devtools/symbol-tool.py count-symbols
Count symbols added since v20.11
$ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05
List experimental symbols present in v20.11 and v21.05
$ devtools/symbol-tool.py list-expired --releases v20.11,v21.05
List experimental symbols in libraries only, present since v19.11
$ devtools/symbol-tool.py list-expired --directory lib
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/symbol-tool.py | 566 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 566 insertions(+)
create mode 100755 devtools/symbol-tool.py
diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py
new file mode 100755
index 0000000000..a77d8b2442
--- /dev/null
+++ b/devtools/symbol-tool.py
@@ -0,0 +1,566 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+# pylint: disable=invalid-name
+'''Tool to count or list symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import re
+import datetime
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ sys.exit()
+
+DESCRIPTION = '''
+This script tracks the growth of stable and experimental symbols
+over releases since v19.11. The script has the ability to
+count the added symbols between two dpdk releases, and to
+list experimental symbols present in two dpdk releases
+(expired symbols), including the name & email of the original contributor.
+
+example usages:
+
+Count symbols added since v19.11
+$ {s} count-symbols
+
+Count symbols added since v20.11
+$ {s} count-symbols --releases v20.11,v21.05
+
+List experimental symbols present in v20.11 and v21.05
+$ {s} list-expired --releases v20.11,v21.05
+
+List experimental symbols in libraries only, present since v19.11
+$ {s} list-expired --directory lib
+'''
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+""" # noqa: E501
+
+
+class EnvironException(Exception):
+ '''Subclass exception for Pylint\'s happiness.'''
+
+
+def get_abi_versions():
+ '''Returns a string of possible dpdk abi versions'''
+
+ year = datetime.date.today().year - 2000
+ tags = " |".join(['\'{}\''.format(i)
+ for i in reversed(range(21, year + 1))])
+ tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return tags
+
+
+def get_dpdk_releases():
+ '''Returns a list of dpdk release tags names since v19.11'''
+
+ year = datetime.date.today().year - 2000
+ year_range = "|".join("{}".format(i) for i in range(19, year + 1))
+ pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ try:
+ result = subprocess.run(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ print("Failed to interogate git for release tags")
+ sys.exit()
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [tag.replace('\"', '')
+ for tag in reversed(tags)
+ if pattern.match(tag)][:-3]
+
+ return tags
+
+
+def fix_directory_name(path):
+ '''Prepend librte to the source directory name'''
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+
+def directory_renamed(path, rel):
+ '''Fix removal of the librte_ from the directory names'''
+
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ return result
+
+
+def mapfile_renamed(path, rel):
+ '''Fix renaming of the map file'''
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree',
+ rel, str(path.parent) + '/'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if newfile is not None:
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+
+def mapfile_and_directory_renamed(path, rel):
+ '''Fix renaming of the map file & the source directory'''
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path), path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+
+FIX_STRATEGIES = [directory_renamed,
+ mapfile_renamed,
+ mapfile_and_directory_renamed]
+
+
+def get_symbols(map_parser, release, mapfile_path):
+ '''Count the symbols for a given release and mapfile'''
+ abi_sections = {}
+
+ tagfile = '{}:{}'.format(release, mapfile_path)
+ try:
+ result = subprocess.run(['git', 'show', tagfile],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ for fix_strategy in FIX_STRATEGIES:
+ if result is not None:
+ break
+ result = fix_strategy(mapfile_path, release)
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = map_parser(mapfile).abi()
+
+ return abi_sections
+
+
+def get_terminal_rows():
+ '''Find the number of rows in the terminal'''
+
+ try:
+ return os.get_terminal_size().lines
+ except IOError:
+ return 0
+
+
+class IgnoredSymbols(): # pylint: disable=too-few-public-methods
+ '''Symbols which are to be ignored for some period'''
+
+ SYMBOL_TOOL_IGNORE = 'devtools/symboltool.abignore'
+ ignore_regex = []
+ __initialized = False
+
+ @staticmethod
+ def initialize():
+ '''initialize once'''
+
+ if IgnoredSymbols.__initialized:
+ return
+ IgnoredSymbols.__initialized = True
+
+ if 'DPDK_SYMBOL_TOOL_IGNORE' in os.environ:
+ IgnoredSymbols.SYMBOL_TOOL_IGNORE = \
+ os.environ['DPDK_SYMBOL_TOOL_IGNORE']
+
+ # if the user specifies an ignore file, we can't find then error.
+ if not Path(IgnoredSymbols.SYMBOL_TOOL_IGNORE).is_file():
+ raise EnvironException('Cannot locate {}\'s '
+ 'ignore file'.format(__file__))
+
+ # if we cannot find the default ignore file, then continue
+ if not Path(IgnoredSymbols.SYMBOL_TOOL_IGNORE).is_file():
+ return
+
+ lines = open(Path(IgnoredSymbols.SYMBOL_TOOL_IGNORE)).readlines()
+ for line in lines:
+
+ line = line.strip()
+
+ # ignore comments and whitespace
+ if line.startswith(';') or len(line) == 0:
+ continue
+
+ IgnoredSymbols.ignore_regex.append(re.compile(line))
+
+ def __init__(self):
+ self.initialize()
+
+ def check_ignore(self, symbol):
+ '''Check symbol against the ignore regexes'''
+
+ for exp in self.ignore_regex:
+ if exp.search(symbol) is not None:
+ return True
+
+ return False
+
+
+class SymbolOwner():
+ '''Find the symbols original contributors name and email'''
+ symbol_regex = {}
+ blame_regex = {'name': r'author\s(.*)',
+ 'email': r'author-mail\s<(.*)>'}
+
+ def __init__(self, libpath, symbol):
+ self.libpath = libpath
+ self.symbol = symbol
+
+ # find variable definitions in C files, and functions in headers.
+ self.symbol_regex = \
+ {'*.c': r'^(?!extern).*' + self.symbol + '[^()]*;',
+ '*.h': r'__rte_experimental(?:.*\n){0,2}.*' + self.symbol}
+
+ def find_symbol_location(self):
+ '''Find where the symbol is definited in the source'''
+ for key in self.symbol_regex:
+ for path in Path(self.libpath).rglob(key):
+ file_text = open(path).read()
+
+ # find where the symbol is defined, either preceded by
+ # rte_experimental tag (functions)
+ # or followed by a ; (variables)
+
+ exp = self.symbol_regex[key]
+ pattern = re.compile(exp, re.MULTILINE)
+ search = pattern.search(file_text)
+
+ if search is not None:
+ symbol_pos = search.span()[1]
+ symbol_line = file_text.count('\n', 0, symbol_pos) + 1
+
+ return [str(path), symbol_line]
+ return None
+
+ def find_symbol_owner(self):
+ '''Find the symbols original contributors name and email'''
+ owners = {}
+ location = self.find_symbol_location()
+
+ if location is None:
+ return None
+
+ line = '-L {},{}'.format(location[1], location[1])
+ # git blame -p(orcelain) -L(ine) path
+ args = ['-p', line, location[0]]
+
+ try:
+ result = subprocess.run(['git', 'blame'] + args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ return None
+
+ blame = result.stdout.decode('utf-8')
+ for key in self.blame_regex:
+ pattern = re.compile(self.blame_regex[key], re.MULTILINE)
+ match = pattern.search(blame)
+
+ owners[key] = match.groups()[0]
+
+ return owners
+
+
+class SymbolCountOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.OUTPUT_FORMATS[format_output](self, dpdk_releases)
+ self.column_titles = ['mapfile'] + dpdk_releases
+
+ self.terminal_rows = get_terminal_rows()
+ self.row = 0
+
+ def set_terminal_output(self, dpdk_rel):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}' + \
+ ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+ self.column_fmt = '{:50}' + \
+ ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+ def set_csv_output(self, dpdk_rel):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},' + \
+ ','.join(['{},{}'] * (len(dpdk_rel)))
+ self.column_fmt = '{},' + \
+ ','.join(['{},'] * (len(dpdk_rel)))
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+ self.row += 1
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+ mapfile = str(mapfile)
+ print(self.output_fmt.format(*([mapfile] + symbols)))
+ self.row += 1
+
+ if((self.terminal_rows > 0) and
+ ((self.row % self.terminal_rows) == 0)):
+ self.print_columns()
+
+ OUTPUT_FORMATS = {None: set_terminal_output,
+ 'terminal': set_terminal_output,
+ 'csv': set_csv_output}
+
+
+class ListExpiredOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.terminal = True
+ self.OUTPUT_FORMATS[format_output](self, dpdk_releases)
+ self.column_titles = ['mapfile'] + \
+ ['expired (' + ','.join(dpdk_releases) + ')'] + \
+ ['contributor name', 'contributor email']
+
+ def set_terminal_output(self, _):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}{:<50}{:<25}{:<25}'
+ self.column_fmt = '{:50}{:50}{:25}{:25}'
+
+ def set_csv_output(self, _):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},{},{},{}'
+ self.column_fmt = '{},{},{},{}'
+ self.terminal = False
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+
+ def print_row(self, mapfile, symbols, owner):
+ '''Print row of symbol values'''
+
+ for symbol in symbols:
+ mapfile = str(mapfile)
+ name = owner[symbol]['name'] \
+ if owner[symbol] is not None else ''
+ email = owner[symbol]['email'] \
+ if owner[symbol] is not None else ''
+
+ print(self.output_fmt.format(mapfile, symbol, name, email))
+ if self.terminal:
+ mapfile = ''
+
+ OUTPUT_FORMATS = {None: set_terminal_output,
+ 'terminal': set_terminal_output,
+ 'csv': set_csv_output}
+
+
+class CountSymbolsAction:
+ ''' Logic to count symbols added since a give release '''
+ IGNORE_SECTIONS = ['EXPERIMENTAL', 'INTERNAL']
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.symbols_count = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbol_count = experimental_count = 0
+
+ symbols = get_symbols(self.parser, release, self.path)
+
+ # which versions are present, and we care about
+ abi_vers = [abi_ver
+ for abi_ver in symbols
+ if abi_ver not in self.IGNORE_SECTIONS]
+
+ for abi_ver in abi_vers:
+ symbol_count += len(symbols[abi_ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental_count = len(symbols['EXPERIMENTAL'])
+
+ self.symbols_count += [symbol_count, experimental_count]
+
+ def __del__(self):
+ self.format_output.print_row(self.path.parent, self.symbols_count)
+
+
+class ListExpiredAction:
+ ''' Logic to list expired symbols between two releases '''
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.experimental_symbols = []
+ self.ignored_symbols = IgnoredSymbols()
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbols = get_symbols(self.parser, release, self.path)
+
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental = [exp.strip() for exp in symbols['EXPERIMENTAL']]
+
+ self.experimental_symbols.append(experimental)
+
+ def __del__(self):
+ if len(self.experimental_symbols) != 2:
+ return
+
+ tmp = self.experimental_symbols
+ # find symbols present in both dpdk releases
+ intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]]
+
+ # remove ignored symbols
+ intersect_syms = [sym for sym in intersect_syms if not
+ self.ignored_symbols.check_ignore(sym)]
+
+ # check for empty set
+ if intersect_syms == []:
+ return
+
+ sym_owner = {}
+ for sym in intersect_syms:
+ sym_owner[sym] = \
+ SymbolOwner(self.path.parent, sym).find_symbol_owner()
+
+ self.format_output.print_row(self.path.parent,
+ intersect_syms,
+ sym_owner)
+
+
+SRC_DIRECTORIES = 'drivers,lib'
+
+ACTIONS = {None: CountSymbolsAction,
+ 'count-symbols': CountSymbolsAction,
+ 'list-expired': ListExpiredAction}
+
+ACTION_OUTPUT = {None: SymbolCountOutput,
+ 'count-symbols': SymbolCountOutput,
+ 'list-expired': ListExpiredOutput}
+
+
+def main():
+ '''Main entry point'''
+
+ dpdk_releases = get_dpdk_releases()
+
+ parser = \
+ argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__),
+ formatter_class=RawTextHelpFormatter)
+
+ parser.add_argument('mode', choices=['count-symbols', 'list-expired'])
+ parser.add_argument('--format-output', choices=['terminal', 'csv'],
+ default='terminal')
+ parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','),
+ default=SRC_DIRECTORIES)
+ parser.add_argument('--releases',
+ help='2 x comma separated release tags e.g. \''
+ + ','.join([dpdk_releases[0], dpdk_releases[-1]])
+ + '\'')
+ args = parser.parse_args()
+
+ if args.releases is not None:
+ dpdk_releases = args.releases.split(',')
+
+ if args.mode == 'list-expired':
+ if len(dpdk_releases) < 2:
+ sys.exit('Please specify two releases to compare '
+ 'in \'list-expired\' mode.')
+ dpdk_releases = [dpdk_releases[0],
+ dpdk_releases[len(dpdk_releases) - 1]]
+
+ action = ACTIONS[args.mode]
+ format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases)
+
+ map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+ map_parser = makeGrammar(map_grammar, {})
+
+ format_output.print_columns()
+
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ release_action = action(path, map_parser, format_output)
+
+ for release in dpdk_releases:
+ release_action.add_mapfile(release)
+
+ # all the magic happens in the destructor
+ del release_action
+
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v13 2/4] devtools: script to send notifications of expired symbols
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 0/4] devtools: scripts to count and track symbols Ray Kinsella
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 1/4] devtools: script to track symbols over releases Ray Kinsella
@ 2021-09-09 13:48 ` Ray Kinsella
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 3/4] maintainers: add new abi scripts Ray Kinsella
` (2 subsequent siblings)
4 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-09 13:48 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
Use this script with the output of the DPDK symbol tool, to notify
maintainers of expired symbols by email. You need to define the environment
variable DPDK_GETMAINTAINER_PATH for this tool to work.
Use terminal output to review the emails before sending.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
devtools/notify_expired_symbols.py --format-output terminal
Then use email output to send the emails to the maintainers.
e.g.
$ devtools/symbol-tool.py list-expired --format-output csv \
| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \
devtools/notify_expired_symbols.py --format-output email \
--smtp-server <server> --sender <someone@somewhere.com> \
--password <password> --cc <someone@somewhere.com>
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
devtools/notify-symbol-maintainers.py | 302 ++++++++++++++++++++++++++
1 file changed, 302 insertions(+)
create mode 100755 devtools/notify-symbol-maintainers.py
diff --git a/devtools/notify-symbol-maintainers.py b/devtools/notify-symbol-maintainers.py
new file mode 100755
index 0000000000..edf330f88b
--- /dev/null
+++ b/devtools/notify-symbol-maintainers.py
@@ -0,0 +1,302 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+# pylint: disable=invalid-name
+'''Tool to notify maintainers of expired symbols'''
+import os
+import smtplib
+import ssl
+import sys
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import time
+from email.message import EmailMessage
+from pathlib import Path
+
+DESCRIPTION = '''
+Use this script with the output of the DPDK symbol tool, to notify maintainers
+and contributors of expired symbols by email. You need to define the environment
+variable DPDK_GETMAINTAINER_PATH for this tool to work.
+
+Use terminal output to review the emails before sending.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+{s} --format-output terminal
+
+Then use email output to send the emails to the maintainers.
+e.g.
+$ devtools/symbol-tool.py list-expired --format-output csv \\
+| DPDK_GETMAINTAINER_PATH=<somewhere>/get_maintainer.pl \\
+{s} --format-output email \\
+--smtp-server <server> --sender <someone@somewhere.com> --password <password> \\
+--cc <someone@somewhere.com>
+''' # noqa: E501
+
+EMAIL_TEMPLATE = '''Hi there,
+
+Please note the symbols listed below have expired. In line with the DPDK ABI
+policy, they should be scheduled for removal, in the next DPDK release.
+
+For more information, please see the DPDK ABI Policy, section 3.5.3.
+https://doc.dpdk.org/guides/contributing/abi_policy.html
+
+Thanks,
+
+The DPDK Symbol Bot
+
+''' # noqa: E501
+
+ABI_POLICY = 'doc/guides/contributing/abi_policy.rst'
+DPDK_GMP_ENV_VAR = 'DPDK_GETMAINTAINER_PATH'
+MAINTAINERS = 'MAINTAINERS'
+get_maintainer = ['devtools/get-maintainer.sh',
+ '--email', '-f']
+
+
+class EnvironException(Exception):
+ '''Subclass exception for Pylint\'s happiness.'''
+
+
+def _die_on_exception(e):
+ '''Print an exception, and quit'''
+
+ print('Fatal Error: ' + str(e))
+ sys.exit()
+
+
+def _check_get_maintainers_env():
+ '''Check get maintainers scripts are setup'''
+
+ if not Path(get_maintainer[0]).is_file():
+ raise EnvironException('Cannot locate DPDK\'s get maintainers script, '
+ ' usually at $' + get_maintainer[0] + '.')
+
+ if DPDK_GMP_ENV_VAR not in os.environ:
+ raise EnvironException(DPDK_GMP_ENV_VAR + ' is not defined.')
+
+ if not Path(os.environ[DPDK_GMP_ENV_VAR]).is_file():
+ raise EnvironException('Cannot locate get maintainers script, usually'
+ ' at ' + DPDK_GMP_ENV_VAR + '.')
+
+
+def _get_maintainers(libpath):
+ '''Get the maintainers for given library'''
+
+ try:
+ _check_get_maintainers_env()
+ except EnvironException as e:
+ _die_on_exception(e)
+
+ try:
+ cmd = get_maintainer + [libpath]
+ result = subprocess.run(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError as e:
+ _die_on_exception(e)
+
+ if result is None:
+ return None
+
+ email = result.stdout.decode('utf-8')
+ if email == '':
+ return None
+
+ email = list(filter(None, email.split('\n')))
+ return email
+
+
+default_maintainers = _get_maintainers(ABI_POLICY) + \
+ _get_maintainers(MAINTAINERS)
+
+
+def get_maintainers(libpath):
+ '''Get the maintainers for given library'''
+ maintainers = _get_maintainers(libpath)
+
+ if maintainers is None:
+ maintainers = default_maintainers
+
+ return maintainers
+
+
+def get_message(library, symbols, config):
+ '''Build email message from symbols, config and maintainers'''
+ contributors = {}
+ message = {}
+ maintainers = get_maintainers(library)
+
+ if maintainers != default_maintainers:
+ message['CC'] = default_maintainers.copy()
+
+ if 'CC' in config:
+ message.setdefault('CC', []).append(config['CC'])
+
+ message['Subject'] = 'Expired symbols in {}\n'.format(library)
+
+ body = EMAIL_TEMPLATE
+ body += '{:<50}{:<25}{:<25}\n'.format('Symbol', 'Contributor', 'Email')
+ for sym in symbols:
+ body += ('{:<50}{:<25}{:<25}\n'.format(sym,
+ symbols[sym]['name'],
+ symbols[sym]['email']))
+ email = symbols[sym]['email']
+ contributors[email] = ''
+
+ contributors = list(contributors.keys())
+
+ message['To'] = maintainers + contributors
+ message['Body'] = body
+
+ return message
+
+
+class OutputEmail():
+ '''Format the output for email'''
+
+ def __init__(self, config):
+ self.config = config
+
+ self.terminal = OutputTerminal(config)
+ context = ssl.create_default_context()
+
+ # Try to log in to server and send email
+ try:
+ self.server = smtplib.SMTP(config['smtp_server'], 587)
+ self.server.starttls(context=context) # Secure the connection
+ self.server.login(config['sender'], config['password'])
+ except EnvironException as e:
+ _die_on_exception(e)
+
+ def message(self, message):
+ '''send email'''
+ self.terminal.message(message)
+
+ msg = EmailMessage()
+ msg.set_content(message.pop('Body'))
+
+ for key in message.keys():
+ msg[key] = message[key]
+
+ msg['From'] = self.config['sender']
+ msg['Reply-To'] = 'no-reply@dpdk.org'
+
+ self.server.send_message(msg)
+
+ time.sleep(1)
+
+ def __del__(self):
+ self.server.quit()
+
+
+class OutputTerminal(): # pylint: disable=too-few-public-methods
+ '''Format the output for the terminal'''
+
+ def __init__(self, config):
+ self.config = config
+
+ def message(self, message):
+ '''Print email to terminal'''
+
+ terminal = 'To:' + ', '.join(message['To']) + '\n'
+ if 'sender' in self.config.keys():
+ terminal += 'From:' + self.config['sender'] + '\n'
+
+ terminal += 'Reply-To:' + 'no-reply@dpdk.org' + '\n'
+
+ if 'CC' in message:
+ terminal += 'CC:' + ', '.join(message['CC']) + '\n'
+
+ terminal += 'Subject:' + message['Subject'] + '\n'
+ terminal += 'Body:' + message['Body'] + '\n'
+
+ print(terminal)
+ print('-' * 80)
+
+
+def parse_config(args):
+ '''put the command line args in the right places'''
+ config = {}
+ error_msg = None
+
+ outputs = {
+ None: OutputTerminal,
+ 'terminal': OutputTerminal,
+ 'email': OutputEmail
+ }
+
+ if args.format_output == 'email':
+ if args.smtp_server is None:
+ error_msg = 'SMTP server'
+ else:
+ config['smtp_server'] = args.smtp_server
+
+ if args.sender is None:
+ error_msg = 'sender'
+ else:
+ config['sender'] = args.sender
+
+ if args.password is None:
+ error_msg = 'password'
+ else:
+ config['password'] = args.password
+
+ if args.cc is not None:
+ config['CC'] = args.cc
+
+ if error_msg is not None:
+ print('Please specify a {} for email output'.format(error_msg))
+ return None
+
+ config['output'] = outputs[args.format_output]
+ return config
+
+
+def main():
+ '''Main entry point'''
+ parser = \
+ argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__),
+ formatter_class=RawTextHelpFormatter)
+ parser.add_argument('--format-output',
+ choices=['terminal', 'email'],
+ default='terminal')
+ parser.add_argument('--smtp-server')
+ parser.add_argument('--password')
+ parser.add_argument('--sender')
+ parser.add_argument('--cc')
+
+ args = parser.parse_args()
+ config = parse_config(args)
+ if config is None:
+ return
+
+ symbols = {}
+ lastlib = library = ''
+
+ output = config['output'](config)
+
+ for line in sys.stdin:
+ line = line.rstrip('\n')
+
+ if line.find('mapfile') >= 0:
+ continue
+ library, symbol, name, email = line.split(',')
+
+ if library != lastlib:
+ message = get_message(lastlib, symbols, config)
+ output.message(message)
+ symbols = {}
+
+ lastlib = library
+ symbols[symbol] = {'name': name, 'email': email}
+
+ # print the last library
+ message = get_message(lastlib, symbols, config)
+ output.message(message)
+
+
+if __name__ == '__main__':
+ main()
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v13 3/4] maintainers: add new abi scripts
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 0/4] devtools: scripts to count and track symbols Ray Kinsella
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 1/4] devtools: script to track symbols over releases Ray Kinsella
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 2/4] devtools: script to send notifications of expired symbols Ray Kinsella
@ 2021-09-09 13:48 ` Ray Kinsella
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 4/4] devtools: add asym crypto to symbol-tool ignore Ray Kinsella
2023-07-06 19:13 ` [dpdk-dev] [PATCH v13 0/4] devtools: scripts to count and track symbols Stephen Hemminger
4 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-09 13:48 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
Add new abi management scripts to the MAINTAINERS file.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
---
MAINTAINERS | 3 +++
1 file changed, 3 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 266f5ac1da..ae38af1b85 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -127,8 +127,11 @@ F: devtools/check-abi-version.sh
F: devtools/check-symbol-change.sh
F: devtools/gen-abi.sh
F: devtools/libabigail.abignore
+F: devtools/symboltool.abignore
F: devtools/update-abi.sh
F: devtools/update_version_map_abi.py
+F: devtools/notify-symbol-maintainers.py
+F: devtools/symbol-tool.py
F: buildtools/check-symbols.sh
F: buildtools/map-list-symbol.sh
F: drivers/*/*/*.map
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* [dpdk-dev] [PATCH v13 4/4] devtools: add asym crypto to symbol-tool ignore
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 0/4] devtools: scripts to count and track symbols Ray Kinsella
` (2 preceding siblings ...)
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 3/4] maintainers: add new abi scripts Ray Kinsella
@ 2021-09-09 13:48 ` Ray Kinsella
2023-07-06 19:13 ` [dpdk-dev] [PATCH v13 0/4] devtools: scripts to count and track symbols Stephen Hemminger
4 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2021-09-09 13:48 UTC (permalink / raw)
To: dev
Cc: bruce.richardson, stephen, ferruh.yigit, thomas, ktraynor, mdr,
aconole, roy.fan.zhang, arkadiuszx.kusztal, gakhil
Add asym crypto to the symbol-tool's ignore file.
Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
Acked-by: Akhil Goyal <gakhil@marvell.com>
---
devtools/symboltool.abignore | 3 +++
1 file changed, 3 insertions(+)
create mode 100644 devtools/symboltool.abignore
diff --git a/devtools/symboltool.abignore b/devtools/symboltool.abignore
new file mode 100644
index 0000000000..800c500a82
--- /dev/null
+++ b/devtools/symboltool.abignore
@@ -0,0 +1,3 @@
+; regex of symbols for the symbol-tool to ignore
+
+rte_cryptodev_asym_.*
--
2.26.2
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v13 0/4] devtools: scripts to count and track symbols
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 0/4] devtools: scripts to count and track symbols Ray Kinsella
` (3 preceding siblings ...)
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 4/4] devtools: add asym crypto to symbol-tool ignore Ray Kinsella
@ 2023-07-06 19:13 ` Stephen Hemminger
4 siblings, 0 replies; 52+ messages in thread
From: Stephen Hemminger @ 2023-07-06 19:13 UTC (permalink / raw)
To: Ray Kinsella
Cc: dev, bruce.richardson, ferruh.yigit, thomas, ktraynor, aconole,
roy.fan.zhang, arkadiuszx.kusztal, gakhil
On Thu, 9 Sep 2021 14:48:04 +0100
Ray Kinsella <mdr@ashroe.eu> wrote:
> The symbol-tool script reports on the growth of symbols over releases
> and list expired symbols. The notify-symbol-maintainers script
> consumes the input from symbol-tool and generates email notifications
> of expired symbols.
>
> v2: reworked to fix pylint errors
> v3: sent with the correct in-reply-to
> v4: fix typos picked up by the CI
> v5: fix terminal_size & directory args
> v6: added list-expired, to list expired experimental symbols
> v7: fix typo in comments
> v8: added tool to notify maintainers of expired symbols
> v9: removed hardcoded emails addressed and script names
> v10: added ability to identify and notify the original contributors
> v11: addressed feedback from Aaron Conole, including PEP8 errors.
> v12: added symbol-tool ignore functionality, to ignore specific symbols
> v13: renamed symboltool.abignore, typos, added ack from Akhil Goyal
>
> Ray Kinsella (4):
> devtools: script to track symbols over releases
> devtools: script to send notifications of expired symbols
> maintainers: add new abi scripts
> devtools: add asym crypto to symbol-tool ignore
Not sure why this never made it in.
Series-Acked-by: Stephen Hemminger <stephen@networkplumber.org>
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v13 1/4] devtools: script to track symbols over releases
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 1/4] devtools: script to track symbols over releases Ray Kinsella
@ 2024-08-15 16:39 ` Stephen Hemminger
2024-08-16 8:54 ` Ray Kinsella
0 siblings, 1 reply; 52+ messages in thread
From: Stephen Hemminger @ 2024-08-15 16:39 UTC (permalink / raw)
To: Ray Kinsella
Cc: dev, bruce.richardson, ferruh.yigit, thomas, ktraynor, aconole,
roy.fan.zhang, arkadiuszx.kusztal, gakhil
On Thu, 9 Sep 2021 14:48:05 +0100
Ray Kinsella <mdr@ashroe.eu> wrote:
> This script tracks the growth of stable and experimental symbols
> over releases since v19.11. The script has the ability to
> count the added symbols between two dpdk releases, and to
> list experimental symbols present in two dpdk releases
> (expired symbols).
>
> example usages:
>
> Count symbols added since v19.11
> $ devtools/symbol-tool.py count-symbols
>
> Count symbols added since v20.11
> $ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05
>
> List experimental symbols present in v20.11 and v21.05
> $ devtools/symbol-tool.py list-expired --releases v20.11,v21.05
>
> List experimental symbols in libraries only, present since v19.11
> $ devtools/symbol-tool.py list-expired --directory lib
>
> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
Tried it, looks useful. Minor notes:
- requires python parsley library which was not required before
- minor style format
$ flake8 ./devtools/symbol-tool.py
./devtools/symbol-tool.py:369:11: E275 missing whitespace after keyword
Acked-by: Stephen Hemminger <stephen@networkplumber.org>
^ permalink raw reply [flat|nested] 52+ messages in thread
* Re: [dpdk-dev] [PATCH v13 1/4] devtools: script to track symbols over releases
2024-08-15 16:39 ` Stephen Hemminger
@ 2024-08-16 8:54 ` Ray Kinsella
0 siblings, 0 replies; 52+ messages in thread
From: Ray Kinsella @ 2024-08-16 8:54 UTC (permalink / raw)
To: Stephen Hemminger
Cc: dev, bruce.richardson, ferruh.yigit, thomas, ktraynor, aconole,
roy.fan.zhang, arkadiuszx.kusztal, gakhil
Stephen Hemminger <stephen@networkplumber.org> writes:
> On Thu, 9 Sep 2021 14:48:05 +0100
> Ray Kinsella <mdr@ashroe.eu> wrote:
>
>> This script tracks the growth of stable and experimental symbols
>> over releases since v19.11. The script has the ability to
>> count the added symbols between two dpdk releases, and to
>> list experimental symbols present in two dpdk releases
>> (expired symbols).
>>
>> example usages:
>>
>> Count symbols added since v19.11
>> $ devtools/symbol-tool.py count-symbols
>>
>> Count symbols added since v20.11
>> $ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05
>>
>> List experimental symbols present in v20.11 and v21.05
>> $ devtools/symbol-tool.py list-expired --releases v20.11,v21.05
>>
>> List experimental symbols in libraries only, present since v19.11
>> $ devtools/symbol-tool.py list-expired --directory lib
>>
>> Signed-off-by: Ray Kinsella <mdr@ashroe.eu>
>
> Tried it, looks useful. Minor notes:
I think it was mostly lack of interest that this didn't get over the
line when I submitted it orginally.
The flake8 errors as easily fixed, I can do that if you thought it might
be useful?
> - requires python parsley library which was not required before
> - minor style format
>
> $ flake8 ./devtools/symbol-tool.py
> ./devtools/symbol-tool.py:369:11: E275 missing whitespace after keyword
>
> Acked-by: Stephen Hemminger <stephen@networkplumber.org>
--
Regards, Ray K
^ permalink raw reply [flat|nested] 52+ messages in thread
end of thread, other threads:[~2024-08-16 8:57 UTC | newest]
Thread overview: 52+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-06-18 16:36 [dpdk-dev] [PATCH] devtools: script to track map symbols Ray Kinsella
2021-06-18 19:40 ` Stephen Hemminger
2021-06-21 9:18 ` Kinsella, Ray
2021-06-21 15:11 ` Ray Kinsella
2021-06-21 15:25 ` [dpdk-dev] [PATCH v3] " Ray Kinsella
2021-06-21 15:35 ` [dpdk-dev] [PATCH v4] " Ray Kinsella
2021-06-21 18:14 ` Stephen Hemminger
2021-06-22 10:19 ` [dpdk-dev] [PATCH v5] " Ray Kinsella
2021-08-04 16:23 ` [dpdk-dev] [PATCH v6] " Ray Kinsella
2021-08-04 16:27 ` [dpdk-dev] [PATCH v7] " Ray Kinsella
2021-08-06 17:54 ` [dpdk-dev] [PATCH v8 0/2] devtools: scripts to count and track symbols Ray Kinsella
2021-08-06 17:54 ` [dpdk-dev] [PATCH v8 1/2] devtools: script to track map symbols Ray Kinsella
2021-08-06 17:54 ` [dpdk-dev] [PATCH v8 2/2] devtools: script to send notifications of expired symbols Ray Kinsella
2021-08-09 12:53 ` [dpdk-dev] [PATCH v9 0/2] devtools: scripts to count and track symbols Ray Kinsella
2021-08-09 12:53 ` [dpdk-dev] [PATCH v9 1/2] devtools: script to track symbols over releases Ray Kinsella
2021-08-09 12:53 ` [dpdk-dev] [PATCH v9 2/2] devtools: script to send notifications of expired symbols Ray Kinsella
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols Ray Kinsella
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 1/3] devtools: script to track symbols over releases Ray Kinsella
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 2/3] devtools: script to send notifications of expired symbols Ray Kinsella
2021-09-01 12:46 ` Aaron Conole
2021-09-03 11:15 ` Kinsella, Ray
2021-09-03 13:32 ` Kinsella, Ray
2021-09-01 13:01 ` David Marchand
2021-09-03 13:28 ` Kinsella, Ray
2021-09-03 13:34 ` Kinsella, Ray
2021-08-31 14:50 ` [dpdk-dev] [PATCH v10 3/3] maintainers: add new abi scripts Ray Kinsella
2021-09-01 12:31 ` [dpdk-dev] [PATCH v10 0/3] devtools: scripts to count and track symbols Aaron Conole
2021-09-01 17:17 ` Stephen Hemminger
2021-09-01 19:04 ` Aaron Conole
2021-09-03 11:17 ` Kinsella, Ray
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 " Ray Kinsella
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 1/3] devtools: script to track symbols over releases Ray Kinsella
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 2/3] devtools: script to send notifications of expired symbols Ray Kinsella
2021-09-03 13:23 ` [dpdk-dev] [PATCH v11 3/3] maintainers: add new abi scripts Ray Kinsella
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 0/3] devtools: scripts to count and track symbols Ray Kinsella
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 1/3] devtools: script to track symbols over releases Ray Kinsella
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 2/3] devtools: script to send notifications of expired symbols Ray Kinsella
2021-09-08 15:12 ` [dpdk-dev] [PATCH v11 3/3] maintainers: add new abi scripts Ray Kinsella
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 0/4] devtools: scripts to count and track symbols Ray Kinsella
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 1/4] devtools: script to track symbols over releases Ray Kinsella
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 2/4] devtools: script to send notifications of expired symbols Ray Kinsella
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 3/4] maintainers: add new abi scripts Ray Kinsella
2021-09-08 15:13 ` [dpdk-dev] [PATCH v12 4/4] devtools: add asym crypto to symbol-tool ignore Ray Kinsella
2021-09-08 15:23 ` [dpdk-dev] [EXT] " Akhil Goyal
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 0/4] devtools: scripts to count and track symbols Ray Kinsella
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 1/4] devtools: script to track symbols over releases Ray Kinsella
2024-08-15 16:39 ` Stephen Hemminger
2024-08-16 8:54 ` Ray Kinsella
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 2/4] devtools: script to send notifications of expired symbols Ray Kinsella
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 3/4] maintainers: add new abi scripts Ray Kinsella
2021-09-09 13:48 ` [dpdk-dev] [PATCH v13 4/4] devtools: add asym crypto to symbol-tool ignore Ray Kinsella
2023-07-06 19:13 ` [dpdk-dev] [PATCH v13 0/4] devtools: scripts to count and track symbols Stephen Hemminger
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).