Owen Hilyard <ohilyard@iol.unh.edu> writes:
> Please ignore this patch. I messed up directories and this patch was not created from the correct file.
Is there an update for this that I missed?
> On Thu, Jan 14, 2021 at 12:59 PM <ohilyard@iol.unh.edu> wrote:
>
> Fixed parsing bugs with the tag parsing script
> Parsing the patch file directly proved to be too unreliabler, so the approach to patch parsing was changed so that
> it relies instead on using a directory with those patches already applied in them and then use a git diff command to get
> the changed files. This avoids needing to parse the patch files directly and should be more reliable.
>
> Added a tool to create an execution file from tags using another execution file as a template
> This tool is intended to allow a list of patch tags to be used in combination with an execution file containing
> information on crbs to only change which test suites are run. This tool enables using the mappings in the
> tests_for_tag.cfg file to run only necessary tests. At present, the mappings represent how tests are currently run at the
> UNH-IOL Community Lab, with the same tests being run for every patch, with the requested change from bug 511
> where documentation changes do not cause any testing.
>
> Signed-off-by: Owen Hilyard <ohilyard@iol.unh.edu>
> ---
> config/tests_for_tag.cfg | 19 ++++++++++++--
> tools/patch_parser.py | 54 ++++++++++++++++++++++++----------------
> 2 files changed, 49 insertions(+), 24 deletions(-)
>
> diff --git a/config/tests_for_tag.cfg b/config/tests_for_tag.cfg
> index 77f797f..7d95c4a 100644
> --- a/config/tests_for_tag.cfg
> +++ b/config/tests_for_tag.cfg
> @@ -5,11 +5,26 @@ core = dynamic_config,
> mtu_update,
> scatter,
> stats_checks,
> + unit_tests_mbuf
> +driver = dynamic_config,
> + link_status_interrupt,
> + mac_filter,
> + mtu_update,
> + scatter,
> + stats_checks,
> + unit_tests_mbuf
> +application = dynamic_config,
> + link_status_interrupt,
> + mac_filter,
> + mtu_update,
> + scatter,
> stats_checks,
> unit_tests_mbuf
> -
> ; Nothing in documentation
> documentation =
>
> [performance]
> -core = nic_single_core_perf,
> +core = nic_single_core_perf
> +driver = nic_single_core_perf
> +application = nic_single_core_perf
> +documentation =
> diff --git a/tools/patch_parser.py b/tools/patch_parser.py
> index 80b8194..73df91f 100755
> --- a/tools/patch_parser.py
> +++ b/tools/patch_parser.py
> @@ -31,30 +31,39 @@
> # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> -
> +import subprocess
>
> import itertools
> from configparser import ConfigParser
> from typing import List, Dict, Set
> import argparse
> +import re
>
> def get_patch_files(patch_file: str) -> List[str]:
> + file_match_pattern = re.compile(r"[\w\-\/.]+ +\| +\d+ [+\-]+")
> with open(patch_file, 'r') as f:
> - lines = list(itertools.takewhile(
> - lambda line: line.strip().endswith('+') or line.strip().endswith('-'),
> - itertools.dropwhile(
> - lambda line: not line.strip().startswith("---"),
> - f.readlines()
> + file_lines = f.readlines()
> + lines = list(filter(
> + lambda line: file_match_pattern.match(line.strip()),
> + itertools.takewhile(
> + lambda line: not line.startswith('---'),
> + list(
> + itertools.dropwhile(
> + lambda line: not line.strip().startswith("---"),
> + file_lines
> + )
> + )[1:]
> )
> ))
> filenames = map(lambda line: line.strip().split(' ')[0], lines)
> - # takewhile includes the --- which starts the filenames
> - return list(filenames)[1:]
> + return list(filenames)
>
> def get_all_files_from_patches(patch_files: List[str]) -> Set[str]:
> - return set(itertools.chain.from_iterable(map(get_patch_files, patch_files)))
> + num_patch_files: int = len(patch_files)
> + files = subprocess.check_output(f"cd dpdk && git diff --name-only HEAD~{num_patch_files}", shell=True).decode
> ('utf-8').splitlines()
> + return set(files)
>
> def parse_comma_delimited_list_from_string(mod_str: str) -> List[str]:
> @@ -80,21 +89,22 @@ def get_tags_for_patches(patch_files: Set[str], dir_attrs: Dict[str, Set[str]])
> ))
>
> -parser = argparse.ArgumentParser(
> - description='Takes a patch file and a config file and creates a list of tags for that patch')
> -parser.add_argument('config_file_path', help='The path to patch_parser.cfg', default='config/patch_parser.cfg')
> -parser.add_argument('patch_file_paths', help='A list of patch files', type=str, metavar='patch file', nargs='+')
> +if __name__ == '__main__':
> + parser = argparse.ArgumentParser(
> + description='Takes a patch file and a config file and creates a list of tags for that patch')
> + parser.add_argument('config_file_path', help='The path to patch_parser.cfg', default='config/patch_parser.cfg')
> + parser.add_argument('patch_file_paths', help='A list of patch files', type=str, metavar='patch file', nargs='+')
>
> -args = parser.parse_args()
> + args = parser.parse_args()
>
> -conf_obj = ConfigParser()
> -conf_obj.read(args.config_file_path)
> + conf_obj = ConfigParser()
> + conf_obj.read(args.config_file_path)
>
> -patch_files = get_all_files_from_patches(args.patch_file_paths)
> -dir_attrs = get_dictionary_attributes_from_config_file(conf_obj)
> -priority_list = parse_comma_delimited_list_from_string(conf_obj['Priority']['priority_list'])
> + patch_files = get_all_files_from_patches(args.patch_file_paths)
> + dir_attrs = get_dictionary_attributes_from_config_file(conf_obj)
> + priority_list = parse_comma_delimited_list_from_string(conf_obj['Priority']['priority_list'])
>
> -unordered_tags: Set[str] = get_tags_for_patches(patch_files, dir_attrs)
> -ordered_tags: List[str] = [tag for tag in priority_list if tag in unordered_tags]
> + unordered_tags: Set[str] = get_tags_for_patches(patch_files, dir_attrs)
> + ordered_tags: List[str] = [tag for tag in priority_list if tag in unordered_tags]
>
> -print("\n".join(ordered_tags))
> + print("\n".join(ordered_tags))
> --
> 2.27.0