Re: [PATCH] tools: add branch rebase support to recheck script

[zhoumin <zhoumin@loongson.cn>]

Tested-by: Min Zhou <zhoumin@loongson.cn>

On 2025/6/12 4:58AM, Patrick Robb wrote:
> Adding support for key value parameters to the recheck framework. This is
> being done specifically to support the branch rebase feature. With this
> commit, the rerun_requests.json includes a new arguments section which
> currently stores the rebase value, and can store future key value pairs
> in the future. This commit does not add a requirement that the user uses
> the rebase argument. It is optional.
>
> There are also some small quality of life changes which have been added.
>
> Signed-off-by: Adam Hassick <ahassick@iol.unh.edu>
> Signed-off-by: Patrick Robb <probb@iol.unh.edu>
> ---
>   tools/get_reruns.py | 151 +++++++++++++++++++++++++-------------------
>   1 file changed, 87 insertions(+), 64 deletions(-)
>
> diff --git a/tools/get_reruns.py b/tools/get_reruns.py
> index ab4d900..7c27650 100755
> --- a/tools/get_reruns.py
> +++ b/tools/get_reruns.py
> @@ -7,17 +7,20 @@ import argparse
>   import datetime
>   import json
>   import re
> -import requests
> -from typing import Dict, List, Optional, Set
> +from json import JSONEncoder
> +from typing import Dict, List, Set, Optional, Tuple
>   
> -DPDK_PATCHWORK_EVENTS_API_URL = "http://patches.dpdk.org/api/events/"
> +import requests
>   
>   
> -class JSONSetEncoder(json.JSONEncoder):
> +class JSONSetEncoder(JSONEncoder):
>       """Custom JSON encoder to handle sets.
>   
>       Pythons json module cannot serialize sets so this custom encoder converts
>       them into lists.
> +
> +    Args:
> +        JSONEncoder: JSON encoder from the json python module.
>       """
>   
>       def default(self, input_object):
> @@ -33,12 +36,10 @@ class RerunProcessor:
>       The idea of this class is to use regex to find certain patterns that
>       represent desired contexts to rerun.
>   
> -    Args:
> +    Arguments:
>           desired_contexts: List of all contexts to search for in the bodies of
>               the comments
>           time_since: Get all comments since this timestamp
> -        pw_api_url: URL for events endpoint of the patchwork API to use for collecting
> -            comments and comment data
>   
>       Attributes:
>           collection_of_retests: A dictionary that maps patch series IDs to the
> @@ -47,38 +48,45 @@ class RerunProcessor:
>           last_comment_timestamp: timestamp of the most recent comment that was
>               processed
>       """
> +    _VALID_ARGS: Set[str] = set(["rebase"])
>   
>       _desired_contexts: List[str]
>       _time_since: str
> -    _pw_api_url: str
>       collection_of_retests: Dict[str, Dict[str, Set]] = {}
>       last_comment_timestamp: Optional[str] = None
> -    # The tag we search for in comments must appear at the start of the line
> -    # and is case sensitive. After this tag we expect a comma separated list
> -    # of valid DPDK patchwork contexts.
> -    #
> +    # ^ is start of line
> +    # ((?:(?:[\\w-]+=)?[\\w-]+(?:, ?\n?)?)+) is a capture group that gets all
> +    #   test labels and key-value pairs after "Recheck-request: "
> +    #   (?:[\\w-]+=)? optionally grabs a key followed by an equals sign
> +    #       (no space)
> +    #       [\\w-] (expanded to "(:?[a-zA-Z0-9-_]+)" ) means 1 more of any
> +    #           character in the ranges a-z, A-Z, 0-9, or the characters
> +    #               '-' or '_'
> +    #       (?:, ?\n?)? means 1 or none of this match group which expects
> +    #           exactly 1 comma followed by 1 or no spaces followed by
> +    #           1 or no newlines.
>       # VALID MATCHES:
>       #   Recheck-request: iol-unit-testing, iol-something-else, iol-one-more,
>       #   Recheck-request: iol-unit-testing,iol-something-else, iol-one-more
>       #   Recheck-request: iol-unit-testing, iol-example, iol-another-example,
>       #   more-intel-testing
> +    #   Recheck-request: x=y, rebase=latest, iol-unit-testing, iol-additional-example
>       # INVALID MATCHES:
>       #   Recheck-request: iol-unit-testing,  intel-example-testing
>       #   Recheck-request: iol-unit-testing iol-something-else,iol-one-more,
> +    #   Recheck-request: iol-unit-testing, rebase = latest
>       #   Recheck-request: iol-unit-testing,iol-something-else,iol-one-more,
> -    #
>       #   more-intel-testing
> -    regex: str = "^Recheck-request: ((?:[a-zA-Z0-9-_]+(?:, ?\n?)?)+)"
> +    regex: str = "^Recheck-request: ((?:(?:[\\w-]+=)?[\\w-]+(?:, ?\n?)?)+)"
> +    last_comment_timestamp: str
>   
> -    def __init__(
> -        self, desired_contexts: List[str], time_since: str, pw_api_url: str
> -    ) -> None:
> +    def __init__(self, desired_contexts: List[str], time_since: str, multipage: bool) -> None:
>           self._desired_contexts = desired_contexts
>           self._time_since = time_since
> -        self._pw_api_url = pw_api_url
> +        self._multipage = multipage
>   
>       def process_reruns(self) -> None:
> -        patchwork_url = f"{self._pw_api_url}?since={self._time_since}"
> +        patchwork_url = f"http://patches.dpdk.org/api/events/?since={self._time_since}"
>           comment_request_info = []
>           for item in [
>               "&category=cover-comment-created",
> @@ -87,6 +95,12 @@ class RerunProcessor:
>               response = requests.get(patchwork_url + item)
>               response.raise_for_status()
>               comment_request_info.extend(response.json())
> +
> +            while 'next' in response.links and self._multipage:
> +                response = requests.get(response.links['next']['url'])
> +                response.raise_for_status()
> +                comment_request_info.extend(response.json())
> +
>           rerun_processor.process_comment_info(comment_request_info)
>   
>       def process_comment_info(self, list_of_comment_blobs: List[Dict]) -> None:
> @@ -138,54 +152,69 @@ class RerunProcessor:
>               comment_info.raise_for_status()
>               content = comment_info.json()["content"]
>   
> -            labels_to_rerun = self.get_test_names(content)
> +            (args, labels_to_rerun) = self.get_test_names_and_parameters(content)
> +
> +            # Accept either filtered labels or arguments.
> +            if labels_to_rerun or (args and self._VALID_ARGS.issuperset(args.keys())):
> +                # Get or insert a new retest request into the dict.
> +                self.collection_of_retests[patch_id] = \
> +                    self.collection_of_retests.get(
> +                        patch_id, {"contexts": set(), "arguments": dict()}
> +                    )
>   
> -            # appending to the list if it already exists, or creating it if it
> -            # doesn't
> -            if labels_to_rerun:
> -                self.collection_of_retests[patch_id] = self.collection_of_retests.get(
> -                    patch_id, {"contexts": set()}
> -                )
> -                self.collection_of_retests[patch_id]["contexts"].update(labels_to_rerun)
> +                req = self.collection_of_retests[patch_id]
>   
> -    def get_test_names(self, email_body: str) -> Set[str]:
> +                # Update the fields.
> +                req["contexts"].update(labels_to_rerun)
> +                req["arguments"].update(args)
> +
> +    def get_test_names_and_parameters(
> +        self, email_body: str
> +    ) -> Tuple[Dict[str, str], Set[str]]:
>           """Uses the regex in the class to get the information from the email.
>   
> -        When it gets the test names from the email, it will all be in one
> -        capture group. We expect a comma separated list of patchwork labels
> -        to be retested.
> +        When it gets the test names from the email, it will be split into two
> +        capture groups. We expect a comma separated list of patchwork labels
> +        to be retested, and another comma separated list of key-value pairs
> +        which are arguments for the retest.
>   
>           Returns:
>               A set of contexts found in the email that match your list of
>               desired contexts to capture. We use a set here to avoid duplicate
>               contexts.
>           """
> -        rerun_section = re.findall(self.regex, email_body, re.MULTILINE)
> -        if not rerun_section:
> -            return set()
> -        rerun_list = list(map(str.strip, rerun_section[0].split(",")))
> -        return set(filter(lambda x: x and x in self._desired_contexts, rerun_list))
> +        rerun_list: Set[str] = set()
> +        params_dict: Dict[str, str] = dict()
> +
> +        match: List[str] = re.findall(self.regex, email_body, re.MULTILINE)
> +        if match:
> +            items: List[str] = list(map(str.strip, match[0].split(",")))
> +
> +            for item in items:
> +                if '=' in item:
> +                    sides = item.split('=')
> +                    params_dict[sides[0]] = sides[1]
> +                else:
> +                    rerun_list.add(item)
> +
> +        return (params_dict, set(filter(lambda x: x in self._desired_contexts, rerun_list)))
>   
> -    def write_output(self, file_name: str) -> None:
> -        """Output class information.
> +    def write_to_output_file(self, file_name: str) -> None:
> +        """Write class information to a JSON file.
>   
>           Takes the collection_of_retests and last_comment_timestamp and outputs
> -        them into either a json file or stdout.
> +        them into a json file.
>   
>           Args:
> -            file_name: Name of the file to write the output to. If this is set
> -            to "-" then it will output to stdout.
> +            file_name: Name of the file to write the output to.
>           """
>   
>           output_dict = {
>               "retests": self.collection_of_retests,
>               "last_comment_timestamp": self.last_comment_timestamp,
>           }
> -        if file_name == "-":
> -            print(json.dumps(output_dict, indent=4, cls=JSONSetEncoder))
> -        else:
> -            with open(file_name, "w") as file:
> -                file.write(json.dumps(output_dict, indent=4, cls=JSONSetEncoder))
> +        with open(file_name, "w") as file:
> +            file.write(json.dumps(output_dict, indent=4, cls=JSONSetEncoder))
>   
>   
>   if __name__ == "__main__":
> @@ -195,39 +224,33 @@ if __name__ == "__main__":
>           "--time-since",
>           dest="time_since",
>           required=True,
> -        help='Get all patches since this timestamp (yyyy-mm-ddThh:mm:ss.SSSSSS).',
> +        help="Get all patches since this many days ago (default: 5)",
>       )
>       parser.add_argument(
>           "--contexts",
>           dest="contexts_to_capture",
>           nargs="*",
>           required=True,
> -        help='List of patchwork contexts you would like to capture.',
> +        help="List of patchwork contexts you would like to capture",
>       )
>       parser.add_argument(
>           "-o",
>           "--out-file",
>           dest="out_file",
>           help=(
> -            'Output file where the list of reruns and the timestamp of the '
> -            'last comment in the list of comments is sent. If this is set '
> -            'to "-" then it will output to stdout (default: -).'
> +            "Output file where the list of reruns and the timestamp of the"
> +            "last comment in the list of comments"
> +            "(default: rerun_requests.json)."
>           ),
> -        default="-",
> +        default="rerun_requests.json",
>       )
>       parser.add_argument(
> -        "-u",
> -        "--patchwork-url",
> -        dest="pw_url",
> -        help=(
> -            'URL for the events endpoint of the patchwork API that will be used to '
> -            f'collect retest requests (default: {DPDK_PATCHWORK_EVENTS_API_URL})'
> -        ),
> -        default=DPDK_PATCHWORK_EVENTS_API_URL
> +        "-m",
> +        "--multipage",
> +        action="store_true",
> +        help="When set, searches all pages of patch/cover comments in the query."
>       )
>       args = parser.parse_args()
> -    rerun_processor = RerunProcessor(
> -        args.contexts_to_capture, args.time_since, args.pw_url
> -    )
> +    rerun_processor = RerunProcessor(args.contexts_to_capture, args.time_since, args.multipage)
>       rerun_processor.process_reruns()
> -    rerun_processor.write_output(args.out_file)
> +    rerun_processor.write_to_output_file(args.out_file)