[dpdk-ci] [PATCH V2] patch-tagging: Added tool to convert tags into dts execution file

[ohilyard@iol.unh.edu]

Fixed parsing bugs with the tag parsing script
	Parsing the patch file directly proved to be too unreliabler, so the approach to patch parsing was changed so that it relies instead on using a directory with those patches already applied in them and then use a git diff command to get the changed files. This avoids needing to parse the patch files directly and should be more reliable.

Added a tool to create an execution file from tags using another execution file as a template
	This tool is intended to allow a list of patch tags to be used in combination with an execution file containing information on crbs to only change which test suites are run. This tool enables using the mappings in the tests_for_tag.cfg file to run only necessary tests. At present, the mappings represent how tests are currently run at the UNH-IOL Community Lab, with the same tests being run for every patch, with the requested change from bug 511 where documentation changes do not cause any testing.

Signed-off-by: Owen Hilyard <ohilyard@iol.unh.edu>
---
 config/tests_for_tag.cfg | 19 ++++++++++++--
 tools/patch_parser.py    | 54 ++++++++++++++++++++++++----------------
 2 files changed, 49 insertions(+), 24 deletions(-)

diff --git a/config/tests_for_tag.cfg b/config/tests_for_tag.cfg
index 77f797f..7d95c4a 100644
--- a/config/tests_for_tag.cfg
+++ b/config/tests_for_tag.cfg
@@ -5,11 +5,26 @@ core = dynamic_config,
        mtu_update,
        scatter,
        stats_checks,
+       unit_tests_mbuf
+driver = dynamic_config,
+       link_status_interrupt,
+       mac_filter,
+       mtu_update,
+       scatter,
+       stats_checks,
+       unit_tests_mbuf
+application = dynamic_config,
+       link_status_interrupt,
+       mac_filter,
+       mtu_update,
+       scatter,
        stats_checks,
        unit_tests_mbuf
-
 ; Nothing in documentation
 documentation =
 
 [performance]
-core = nic_single_core_perf,
+core = nic_single_core_perf
+driver = nic_single_core_perf
+application = nic_single_core_perf
+documentation =
diff --git a/tools/patch_parser.py b/tools/patch_parser.py
index 80b8194..73df91f 100755
--- a/tools/patch_parser.py
+++ b/tools/patch_parser.py
@@ -31,30 +31,39 @@
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
+import subprocess
 
 import itertools
 from configparser import ConfigParser
 from typing import List, Dict, Set
 import argparse
+import re
 
 
 def get_patch_files(patch_file: str) -> List[str]:
+    file_match_pattern = re.compile(r"[\w\-\/.]+ +\| +\d+ [+\-]+")
     with open(patch_file, 'r') as f:
-        lines = list(itertools.takewhile(
-            lambda line: line.strip().endswith('+') or line.strip().endswith('-'),
-            itertools.dropwhile(
-                lambda line: not line.strip().startswith("---"),
-                f.readlines()
+        file_lines = f.readlines()
+        lines = list(filter(
+            lambda line: file_match_pattern.match(line.strip()),
+            itertools.takewhile(
+                lambda line: not line.startswith('---'),
+                list(
+                    itertools.dropwhile(
+                        lambda line: not line.strip().startswith("---"),
+                        file_lines
+                    )
+                )[1:]
             )
         ))
         filenames = map(lambda line: line.strip().split(' ')[0], lines)
-        # takewhile includes the --- which starts the filenames
-        return list(filenames)[1:]
+        return list(filenames)
 
 
 def get_all_files_from_patches(patch_files: List[str]) -> Set[str]:
-    return set(itertools.chain.from_iterable(map(get_patch_files, patch_files)))
+    num_patch_files: int = len(patch_files)
+    files = subprocess.check_output(f"cd dpdk && git diff --name-only HEAD~{num_patch_files}", shell=True).decode('utf-8').splitlines()
+    return set(files)
 
 
 def parse_comma_delimited_list_from_string(mod_str: str) -> List[str]:
@@ -80,21 +89,22 @@ def get_tags_for_patches(patch_files: Set[str], dir_attrs: Dict[str, Set[str]])
     ))
 
 
-parser = argparse.ArgumentParser(
-    description='Takes a patch file and a config file and creates a list of tags for that patch')
-parser.add_argument('config_file_path', help='The path to patch_parser.cfg', default='config/patch_parser.cfg')
-parser.add_argument('patch_file_paths', help='A list of patch files', type=str, metavar='patch file', nargs='+')
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Takes a patch file and a config file and creates a list of tags for that patch')
+    parser.add_argument('config_file_path', help='The path to patch_parser.cfg', default='config/patch_parser.cfg')
+    parser.add_argument('patch_file_paths', help='A list of patch files', type=str, metavar='patch file', nargs='+')
 
-args = parser.parse_args()
+    args = parser.parse_args()
 
-conf_obj = ConfigParser()
-conf_obj.read(args.config_file_path)
+    conf_obj = ConfigParser()
+    conf_obj.read(args.config_file_path)
 
-patch_files = get_all_files_from_patches(args.patch_file_paths)
-dir_attrs = get_dictionary_attributes_from_config_file(conf_obj)
-priority_list = parse_comma_delimited_list_from_string(conf_obj['Priority']['priority_list'])
+    patch_files = get_all_files_from_patches(args.patch_file_paths)
+    dir_attrs = get_dictionary_attributes_from_config_file(conf_obj)
+    priority_list = parse_comma_delimited_list_from_string(conf_obj['Priority']['priority_list'])
 
-unordered_tags: Set[str] = get_tags_for_patches(patch_files, dir_attrs)
-ordered_tags: List[str] = [tag for tag in priority_list if tag in unordered_tags]
+    unordered_tags: Set[str] = get_tags_for_patches(patch_files, dir_attrs)
+    ordered_tags: List[str] = [tag for tag in priority_list if tag in unordered_tags]
 
-print("\n".join(ordered_tags))
+    print("\n".join(ordered_tags))
-- 
2.27.0