Re: [PATCH v6 3/3] dts: add performance test functions to test suite API

[Andrew Bailey <abailey@iol.unh.edu>]

[-- Attachment #1: Type: text/plain, Size: 16530 bytes --]

+class Config(BaseConfig):
+    """Performance test metrics."""
+
+    test_parameters: list[dict[str, int | float]] = [
+        {"frame_size": 64, "num_descriptors": 1024, "expected_mpps": 1.00},
+        {"frame_size": 128, "num_descriptors": 1024, "expected_mpps":
1.00},
+        {"frame_size": 256, "num_descriptors": 1024, "expected_mpps":
1.00},
+        {"frame_size": 512, "num_descriptors": 1024, "expected_mpps":
1.00},
+        {"frame_size": 1024, "num_descriptors": 1024, "expected_mpps":
1.00},
+        {"frame_size": 1518, "num_descriptors": 1024, "expected_mpps":
1.00},
+    ]
+    delta_tolerance: float = 0.05

Disregard the last comment, I had assumed that the delta_tolerance was in
mpps
not percentage of expected_mpps.

nit: It may be helpful to declare this distinction in a comment above the
delta_tolerance
variable or perhaps use percent_tolerance instead.

Reviewed-by: Andrew Bailey <abailey@iol.unh.edu>

On Thu, Nov 6, 2025 at 8:30 AM Andrew Bailey <abailey@iol.unh.edu> wrote:

> +                params["measured_mpps"] = self._transmit(testpmd,
> frame_size)
> +                params["performance_delta"] = (
> +                    float(params["measured_mpps"]) -
> float(params["expected_mpps"])
> +                ) / float(params["expected_mpps"])
> +                params["pass"] = float(params["performance_delta"]) >=
> -self.delta_tolerance
>
> This code seems like it can produce false positives. If we are checking if
> a measured mpps is within dela_tolerance of the expected, (pass =
> measured_mpps >= expected_mpps - delta_tolerance) may be more effective. As
> an example of a false positive, use measured_mpps = 1.0, expected_mpps =
> 2.0, and delta_tolerance = 0.51. This should be a fail since 1.0 is not
> within 0.51 of 2.0 but equates to (-.50) >= (-.51) == true.
>
> On Wed, Nov 5, 2025 at 5:37 PM Patrick Robb <probb@iol.unh.edu> wrote:
>
>> From: Nicholas Pratte <npratte@iol.unh.edu>
>>
>> Provide packet transmission function to support performance tests using a
>> user-supplied performance traffic generator. The single core performance
>> test is included. It allows the user to define a matrix of frame size,
>> descriptor count, and expected mpps, and fails if any combination does
>> not forward a mpps count within 5% of the given baseline.
>>
>> Bugzilla ID: 1697
>> Signed-off-by: Nicholas Pratte <npratte@iol.unh.edu>
>> Signed-off-by: Patrick Robb <probb@iol.unh.edu>
>> Reviewed-by: Dean Marx <dmarx@iol.unh.edu>
>> Reviewed-by: Andrew Bailey <abailey@iol.unh.edu>
>> ---
>>  ...sts.TestSuite_single_core_forward_perf.rst |   8 +
>>  dts/api/packet.py                             |  35 +++-
>>  dts/api/test.py                               |  32 ++++
>>  dts/configurations/tests_config.example.yaml  |  12 ++
>>  .../TestSuite_single_core_forward_perf.py     | 149 ++++++++++++++++++
>>  5 files changed, 235 insertions(+), 1 deletion(-)
>>  create mode 100644
>> doc/api/dts/tests.TestSuite_single_core_forward_perf.rst
>>  create mode 100644 dts/tests/TestSuite_single_core_forward_perf.py
>>
>> diff --git a/doc/api/dts/tests.TestSuite_single_core_forward_perf.rst
>> b/doc/api/dts/tests.TestSuite_single_core_forward_perf.rst
>> new file mode 100644
>> index 0000000000..3651b0b041
>> --- /dev/null
>> +++ b/doc/api/dts/tests.TestSuite_single_core_forward_perf.rst
>> @@ -0,0 +1,8 @@
>> +.. SPDX-License-Identifier: BSD-3-Clause
>> +
>> +single_core_forward_perf Test Suite
>> +===================================
>> +
>> +.. automodule:: tests.TestSuite_single_core_forward_perf
>> +   :members:
>> +   :show-inheritance:
>> diff --git a/dts/api/packet.py b/dts/api/packet.py
>> index ac7f64dd17..094a1b7a9d 100644
>> --- a/dts/api/packet.py
>> +++ b/dts/api/packet.py
>> @@ -33,6 +33,9 @@
>>  from
>> framework.testbed_model.traffic_generator.capturing_traffic_generator
>> import (
>>      PacketFilteringConfig,
>>  )
>> +from
>> framework.testbed_model.traffic_generator.performance_traffic_generator
>> import (
>> +    PerformanceTrafficStats,
>> +)
>>  from framework.utils import get_packet_summaries
>>
>>
>> @@ -108,7 +111,9 @@ def send_packets(
>>          packets: Packets to send.
>>      """
>>      packets = adjust_addresses(packets)
>> -    get_ctx().func_tg.send_packets(packets,
>> get_ctx().topology.tg_port_egress)
>> +    tg = get_ctx().func_tg
>> +    if tg:
>> +        tg.send_packets(packets, get_ctx().topology.tg_port_egress)
>>
>>
>>  def get_expected_packets(
>> @@ -317,3 +322,31 @@ def _verify_l3_packet(received_packet: IP,
>> expected_packet: IP) -> bool:
>>      if received_packet.src != expected_packet.src or received_packet.dst
>> != expected_packet.dst:
>>          return False
>>      return True
>> +
>> +
>> +def assess_performance_by_packet(
>> +    packet: Packet, duration: float, send_mpps: int | None = None
>> +) -> PerformanceTrafficStats:
>> +    """Send a given packet for a given duration and assess basic
>> performance statistics.
>> +
>> +    Send `packet` and assess NIC performance for a given duration,
>> corresponding to the test
>> +    suite's given topology.
>> +
>> +    Args:
>> +        packet: The packet to send.
>> +        duration: Performance test duration (in seconds).
>> +        send_mpps: The millions packets per second send rate.
>> +
>> +    Returns:
>> +        Performance statistics of the generated test.
>> +    """
>> +    from
>> framework.testbed_model.traffic_generator.performance_traffic_generator
>> import (
>> +        PerformanceTrafficGenerator,
>> +    )
>> +
>> +    assert isinstance(
>> +        get_ctx().perf_tg, PerformanceTrafficGenerator
>> +    ), "Cannot send performance traffic with non-performance traffic
>> generator"
>> +    tg: PerformanceTrafficGenerator = cast(PerformanceTrafficGenerator,
>> get_ctx().perf_tg)
>> +    # TODO: implement @requires for types of traffic generator
>> +    return tg.calculate_traffic_and_stats(packet, duration, send_mpps)
>> diff --git a/dts/api/test.py b/dts/api/test.py
>> index f58c82715d..11265ee2c1 100644
>> --- a/dts/api/test.py
>> +++ b/dts/api/test.py
>> @@ -6,9 +6,13 @@
>>  This module provides utility functions for test cases, including
>> logging, verification.
>>  """
>>
>> +import json
>> +from datetime import datetime
>> +
>>  from framework.context import get_ctx
>>  from framework.exception import InternalError, SkippedTestException,
>> TestCaseVerifyError
>>  from framework.logger import DTSLogger
>> +from framework.testbed_model.artifact import Artifact
>>
>>
>>  def get_current_test_case_name() -> str:
>> @@ -124,3 +128,31 @@ def get_logger() -> DTSLogger:
>>      if current_test_suite is None:
>>          raise InternalError("No current test suite")
>>      return current_test_suite._logger
>> +
>> +
>> +def write_performance_json(
>> +    performance_data: dict, filename: str = "performance_metrics.json"
>> +) -> None:
>> +    """Write performance test results to a JSON file in the test suite's
>> output directory.
>> +
>> +    This method creates a JSON file containing performance metrics in
>> the test suite's
>> +    output directory. The data can be a dictionary of any structure. No
>> specific format
>> +    is required.
>> +
>> +    Args:
>> +        performance_data: Dictionary containing performance metrics and
>> results.
>> +        filename: Name of the JSON file to create.
>> +
>> +    Raises:
>> +        InternalError: If performance data is not provided.
>> +    """
>> +    if not performance_data:
>> +        raise InternalError("No performance data to write")
>> +
>> +    perf_data = {"timestamp": datetime.now().isoformat(),
>> **performance_data}
>> +    perf_json_artifact = Artifact("local", filename)
>> +
>> +    with perf_json_artifact.open("w") as json_file:
>> +        json.dump(perf_data, json_file, indent=2)
>> +
>> +    get_logger().info(f"Performance results written to:
>> {perf_json_artifact.local_path}")
>> diff --git a/dts/configurations/tests_config.example.yaml
>> b/dts/configurations/tests_config.example.yaml
>> index c011ac0588..167bc91a35 100644
>> --- a/dts/configurations/tests_config.example.yaml
>> +++ b/dts/configurations/tests_config.example.yaml
>> @@ -3,3 +3,15 @@
>>  # Define the custom test suite configurations
>>  hello_world:
>>    msg: A custom hello world to you!
>> +single_core_forward_perf:
>> +  test_parameters: # Add frame size / descriptor count combinations as
>> needed
>> +    - frame_size: 64
>> +      num_descriptors: 512
>> +      expected_mpps: 1.0 # Set millions of packets per second according
>> to your devices expected throughput for this given frame size / descriptor
>> count
>> +    - frame_size: 64
>> +      num_descriptors: 1024
>> +      expected_mpps: 1.0
>> +    - frame_size: 512
>> +      num_descriptors: 1024
>> +      expected_mpps: 1.0
>> +  delta_tolerance: 0.05
>> \ No newline at end of file
>> diff --git a/dts/tests/TestSuite_single_core_forward_perf.py
>> b/dts/tests/TestSuite_single_core_forward_perf.py
>> new file mode 100644
>> index 0000000000..8a92ba39b5
>> --- /dev/null
>> +++ b/dts/tests/TestSuite_single_core_forward_perf.py
>> @@ -0,0 +1,149 @@
>> +# SPDX-License-Identifier: BSD-3-Clause
>> +# Copyright(c) 2025 University of New Hampshire
>> +
>> +"""Single core forwarding performance test suite.
>> +
>> +This suite measures the amount of packets which can be forwarded by DPDK
>> using a single core.
>> +The testsuites takes in as parameters a set of parameters, each
>> consisting of a frame size,
>> +Tx/Rx descriptor count, and the expected MPPS to be forwarded by the
>> DPDK application. The
>> +test leverages a performance traffic generator to send traffic at two
>> paired TestPMD interfaces
>> +on the SUT system, which forward to one another and then back to the
>> traffic generator's ports.
>> +The aggregate packets forwarded by the two TestPMD ports are compared
>> against the expected MPPS
>> +baseline which is given in the test config, in order to determine the
>> test result.
>> +"""
>> +
>> +from scapy.layers.inet import IP
>> +from scapy.layers.l2 import Ether
>> +from scapy.packet import Raw
>> +
>> +from api.capabilities import (
>> +    LinkTopology,
>> +    requires_link_topology,
>> +)
>> +from api.packet import assess_performance_by_packet
>> +from api.test import verify, write_performance_json
>> +from api.testpmd import TestPmd
>> +from api.testpmd.config import RXRingParams, TXRingParams
>> +from framework.params.types import TestPmdParamsDict
>> +from framework.test_suite import BaseConfig, TestSuite, perf_test
>> +
>> +
>> +class Config(BaseConfig):
>> +    """Performance test metrics."""
>> +
>> +    test_parameters: list[dict[str, int | float]] = [
>> +        {"frame_size": 64, "num_descriptors": 1024, "expected_mpps":
>> 1.00},
>> +        {"frame_size": 128, "num_descriptors": 1024, "expected_mpps":
>> 1.00},
>> +        {"frame_size": 256, "num_descriptors": 1024, "expected_mpps":
>> 1.00},
>> +        {"frame_size": 512, "num_descriptors": 1024, "expected_mpps":
>> 1.00},
>> +        {"frame_size": 1024, "num_descriptors": 1024, "expected_mpps":
>> 1.00},
>> +        {"frame_size": 1518, "num_descriptors": 1024, "expected_mpps":
>> 1.00},
>> +    ]
>> +    delta_tolerance: float = 0.05
>> +
>> +
>> +@requires_link_topology(LinkTopology.TWO_LINKS)
>> +class TestSingleCoreForwardPerf(TestSuite):
>> +    """Single core forwarding performance test suite."""
>> +
>> +    config: Config
>> +
>> +    def set_up_suite(self):
>> +        """Set up the test suite."""
>> +        self.test_parameters = self.config.test_parameters
>> +        self.delta_tolerance = self.config.delta_tolerance
>> +
>> +    def _transmit(self, testpmd: TestPmd, frame_size: int) -> float:
>> +        """Create a testpmd session with every rule in the given list,
>> verify jump behavior.
>> +
>> +        Args:
>> +            testpmd: The testpmd shell to use for forwarding packets.
>> +            frame_size: The size of the frame to transmit.
>> +
>> +        Returns:
>> +            The MPPS (millions of packets per second) forwarded by the
>> SUT.
>> +        """
>> +        # Build packet with dummy values, and account for the 14B and
>> 20B Ether and IP headers
>> +        packet = (
>> +            Ether(src="52:00:00:00:00:00")
>> +            / IP(src="1.2.3.4", dst="192.18.1.0")
>> +            / Raw(load="x" * (frame_size - 14 - 20))
>> +        )
>> +
>> +        testpmd.start()
>> +
>> +        # Transmit for 30 seconds.
>> +        stats = assess_performance_by_packet(packet=packet, duration=30)
>> +
>> +        rx_mpps = stats.rx_pps / 1_000_000
>> +
>> +        return rx_mpps
>> +
>> +    def _produce_stats_table(self, test_parameters: list[dict[str, int |
>> float]]) -> None:
>> +        """Display performance results in table format and write to
>> structured JSON file.
>> +
>> +        Args:
>> +            test_parameters: The expected and real stats per set of test
>> parameters.
>> +        """
>> +        header = f"{'Frame Size':>12} | {'TXD/RXD':>12} | {'Real
>> MPPS':>12} | {'Expected MPPS':>14}"
>> +        print("-" * len(header))
>> +        print(header)
>> +        print("-" * len(header))
>> +        for params in test_parameters:
>> +            print(f"{params['frame_size']:>12} |
>> {params['num_descriptors']:>12} | ", end="")
>> +            print(f"{params['measured_mpps']:>12.2f} |
>> {params['expected_mpps']:>14.2f}")
>> +            print("-" * len(header))
>> +
>> +        write_performance_json({"results": test_parameters})
>> +
>> +    @perf_test
>> +    def single_core_forward_perf(self) -> None:
>> +        """Validate expected single core forwarding performance.
>> +
>> +        Steps:
>> +            * Create a packet according to the frame size specified in
>> the test config.
>> +            * Transmit from the traffic generator's ports 0 and 1 at
>> above the expect.
>> +            * Forward on TestPMD's interfaces 0 and 1 with 1 core.
>> +
>> +        Verify:
>> +            * The resulting MPPS forwarded is greater than
>> expected_mpps*(1-delta_tolerance).
>> +        """
>> +        # Find SUT DPDK driver to determine driver specific performance
>> optimization flags
>> +        sut_dpdk_driver =
>> self._ctx.sut_node.config.ports[0].os_driver_for_dpdk
>> +
>> +        for params in self.test_parameters:
>> +            frame_size = params["frame_size"]
>> +            num_descriptors = params["num_descriptors"]
>> +
>> +            driver_specific_testpmd_args: TestPmdParamsDict = {
>> +                "tx_ring": TXRingParams(descriptors=num_descriptors),
>> +                "rx_ring": RXRingParams(descriptors=num_descriptors),
>> +                "nb_cores": 1,
>> +            }
>> +
>> +            if sut_dpdk_driver == "mlx5_core":
>> +                driver_specific_testpmd_args["burst"] = 64
>> +                driver_specific_testpmd_args["mbcache"] = 512
>> +            elif sut_dpdk_driver == "i40e":
>> +                driver_specific_testpmd_args["rx_queues"] = 2
>> +                driver_specific_testpmd_args["tx_queues"] = 2
>> +
>> +            with TestPmd(
>> +                **driver_specific_testpmd_args,
>> +            ) as testpmd:
>> +                params["measured_mpps"] = self._transmit(testpmd,
>> frame_size)
>> +                params["performance_delta"] = (
>> +                    float(params["measured_mpps"]) -
>> float(params["expected_mpps"])
>> +                ) / float(params["expected_mpps"])
>> +                params["pass"] = float(params["performance_delta"]) >=
>> -self.delta_tolerance
>> +
>> +        self._produce_stats_table(self.test_parameters)
>> +
>> +        for params in self.test_parameters:
>> +            verify(
>> +                params["pass"] is True,
>> +                f"""Packets forwarded is less than {(1
>> -self.delta_tolerance)*100}%
>> +                of the expected baseline.
>> +                Measured MPPS = {params["measured_mpps"]}
>> +                Expected MPPS = {params["expected_mpps"]}""",
>> +            )
>> --
>> 2.49.0
>>
>>

[-- Attachment #2: Type: text/html, Size: 19778 bytes --]