+ params["measured_mpps"] = self._transmit(testpmd, frame_size) + params["performance_delta"] = ( + float(params["measured_mpps"]) - float(params["expected_mpps"]) + ) / float(params["expected_mpps"]) + params["pass"] = float(params["performance_delta"]) >= -self.delta_tolerance This code seems like it can produce false positives. If we are checking if a measured mpps is within dela_tolerance of the expected, (pass = measured_mpps >= expected_mpps - delta_tolerance) may be more effective. As an example of a false positive, use measured_mpps = 1.0, expected_mpps = 2.0, and delta_tolerance = 0.51. This should be a fail since 1.0 is not within 0.51 of 2.0 but equates to (-.50) >= (-.51) == true. On Wed, Nov 5, 2025 at 5:37 PM Patrick Robb wrote: > From: Nicholas Pratte > > Provide packet transmission function to support performance tests using a > user-supplied performance traffic generator. The single core performance > test is included. It allows the user to define a matrix of frame size, > descriptor count, and expected mpps, and fails if any combination does > not forward a mpps count within 5% of the given baseline. > > Bugzilla ID: 1697 > Signed-off-by: Nicholas Pratte > Signed-off-by: Patrick Robb > Reviewed-by: Dean Marx > Reviewed-by: Andrew Bailey > --- > ...sts.TestSuite_single_core_forward_perf.rst | 8 + > dts/api/packet.py | 35 +++- > dts/api/test.py | 32 ++++ > dts/configurations/tests_config.example.yaml | 12 ++ > .../TestSuite_single_core_forward_perf.py | 149 ++++++++++++++++++ > 5 files changed, 235 insertions(+), 1 deletion(-) > create mode 100644 > doc/api/dts/tests.TestSuite_single_core_forward_perf.rst > create mode 100644 dts/tests/TestSuite_single_core_forward_perf.py > > diff --git a/doc/api/dts/tests.TestSuite_single_core_forward_perf.rst > b/doc/api/dts/tests.TestSuite_single_core_forward_perf.rst > new file mode 100644 > index 0000000000..3651b0b041 > --- /dev/null > +++ b/doc/api/dts/tests.TestSuite_single_core_forward_perf.rst > @@ -0,0 +1,8 @@ > +.. SPDX-License-Identifier: BSD-3-Clause > + > +single_core_forward_perf Test Suite > +=================================== > + > +.. automodule:: tests.TestSuite_single_core_forward_perf > + :members: > + :show-inheritance: > diff --git a/dts/api/packet.py b/dts/api/packet.py > index ac7f64dd17..094a1b7a9d 100644 > --- a/dts/api/packet.py > +++ b/dts/api/packet.py > @@ -33,6 +33,9 @@ > from > framework.testbed_model.traffic_generator.capturing_traffic_generator > import ( > PacketFilteringConfig, > ) > +from > framework.testbed_model.traffic_generator.performance_traffic_generator > import ( > + PerformanceTrafficStats, > +) > from framework.utils import get_packet_summaries > > > @@ -108,7 +111,9 @@ def send_packets( > packets: Packets to send. > """ > packets = adjust_addresses(packets) > - get_ctx().func_tg.send_packets(packets, > get_ctx().topology.tg_port_egress) > + tg = get_ctx().func_tg > + if tg: > + tg.send_packets(packets, get_ctx().topology.tg_port_egress) > > > def get_expected_packets( > @@ -317,3 +322,31 @@ def _verify_l3_packet(received_packet: IP, > expected_packet: IP) -> bool: > if received_packet.src != expected_packet.src or received_packet.dst > != expected_packet.dst: > return False > return True > + > + > +def assess_performance_by_packet( > + packet: Packet, duration: float, send_mpps: int | None = None > +) -> PerformanceTrafficStats: > + """Send a given packet for a given duration and assess basic > performance statistics. > + > + Send `packet` and assess NIC performance for a given duration, > corresponding to the test > + suite's given topology. > + > + Args: > + packet: The packet to send. > + duration: Performance test duration (in seconds). > + send_mpps: The millions packets per second send rate. > + > + Returns: > + Performance statistics of the generated test. > + """ > + from > framework.testbed_model.traffic_generator.performance_traffic_generator > import ( > + PerformanceTrafficGenerator, > + ) > + > + assert isinstance( > + get_ctx().perf_tg, PerformanceTrafficGenerator > + ), "Cannot send performance traffic with non-performance traffic > generator" > + tg: PerformanceTrafficGenerator = cast(PerformanceTrafficGenerator, > get_ctx().perf_tg) > + # TODO: implement @requires for types of traffic generator > + return tg.calculate_traffic_and_stats(packet, duration, send_mpps) > diff --git a/dts/api/test.py b/dts/api/test.py > index f58c82715d..11265ee2c1 100644 > --- a/dts/api/test.py > +++ b/dts/api/test.py > @@ -6,9 +6,13 @@ > This module provides utility functions for test cases, including logging, > verification. > """ > > +import json > +from datetime import datetime > + > from framework.context import get_ctx > from framework.exception import InternalError, SkippedTestException, > TestCaseVerifyError > from framework.logger import DTSLogger > +from framework.testbed_model.artifact import Artifact > > > def get_current_test_case_name() -> str: > @@ -124,3 +128,31 @@ def get_logger() -> DTSLogger: > if current_test_suite is None: > raise InternalError("No current test suite") > return current_test_suite._logger > + > + > +def write_performance_json( > + performance_data: dict, filename: str = "performance_metrics.json" > +) -> None: > + """Write performance test results to a JSON file in the test suite's > output directory. > + > + This method creates a JSON file containing performance metrics in the > test suite's > + output directory. The data can be a dictionary of any structure. No > specific format > + is required. > + > + Args: > + performance_data: Dictionary containing performance metrics and > results. > + filename: Name of the JSON file to create. > + > + Raises: > + InternalError: If performance data is not provided. > + """ > + if not performance_data: > + raise InternalError("No performance data to write") > + > + perf_data = {"timestamp": datetime.now().isoformat(), > **performance_data} > + perf_json_artifact = Artifact("local", filename) > + > + with perf_json_artifact.open("w") as json_file: > + json.dump(perf_data, json_file, indent=2) > + > + get_logger().info(f"Performance results written to: > {perf_json_artifact.local_path}") > diff --git a/dts/configurations/tests_config.example.yaml > b/dts/configurations/tests_config.example.yaml > index c011ac0588..167bc91a35 100644 > --- a/dts/configurations/tests_config.example.yaml > +++ b/dts/configurations/tests_config.example.yaml > @@ -3,3 +3,15 @@ > # Define the custom test suite configurations > hello_world: > msg: A custom hello world to you! > +single_core_forward_perf: > + test_parameters: # Add frame size / descriptor count combinations as > needed > + - frame_size: 64 > + num_descriptors: 512 > + expected_mpps: 1.0 # Set millions of packets per second according > to your devices expected throughput for this given frame size / descriptor > count > + - frame_size: 64 > + num_descriptors: 1024 > + expected_mpps: 1.0 > + - frame_size: 512 > + num_descriptors: 1024 > + expected_mpps: 1.0 > + delta_tolerance: 0.05 > \ No newline at end of file > diff --git a/dts/tests/TestSuite_single_core_forward_perf.py > b/dts/tests/TestSuite_single_core_forward_perf.py > new file mode 100644 > index 0000000000..8a92ba39b5 > --- /dev/null > +++ b/dts/tests/TestSuite_single_core_forward_perf.py > @@ -0,0 +1,149 @@ > +# SPDX-License-Identifier: BSD-3-Clause > +# Copyright(c) 2025 University of New Hampshire > + > +"""Single core forwarding performance test suite. > + > +This suite measures the amount of packets which can be forwarded by DPDK > using a single core. > +The testsuites takes in as parameters a set of parameters, each > consisting of a frame size, > +Tx/Rx descriptor count, and the expected MPPS to be forwarded by the DPDK > application. The > +test leverages a performance traffic generator to send traffic at two > paired TestPMD interfaces > +on the SUT system, which forward to one another and then back to the > traffic generator's ports. > +The aggregate packets forwarded by the two TestPMD ports are compared > against the expected MPPS > +baseline which is given in the test config, in order to determine the > test result. > +""" > + > +from scapy.layers.inet import IP > +from scapy.layers.l2 import Ether > +from scapy.packet import Raw > + > +from api.capabilities import ( > + LinkTopology, > + requires_link_topology, > +) > +from api.packet import assess_performance_by_packet > +from api.test import verify, write_performance_json > +from api.testpmd import TestPmd > +from api.testpmd.config import RXRingParams, TXRingParams > +from framework.params.types import TestPmdParamsDict > +from framework.test_suite import BaseConfig, TestSuite, perf_test > + > + > +class Config(BaseConfig): > + """Performance test metrics.""" > + > + test_parameters: list[dict[str, int | float]] = [ > + {"frame_size": 64, "num_descriptors": 1024, "expected_mpps": > 1.00}, > + {"frame_size": 128, "num_descriptors": 1024, "expected_mpps": > 1.00}, > + {"frame_size": 256, "num_descriptors": 1024, "expected_mpps": > 1.00}, > + {"frame_size": 512, "num_descriptors": 1024, "expected_mpps": > 1.00}, > + {"frame_size": 1024, "num_descriptors": 1024, "expected_mpps": > 1.00}, > + {"frame_size": 1518, "num_descriptors": 1024, "expected_mpps": > 1.00}, > + ] > + delta_tolerance: float = 0.05 > + > + > +@requires_link_topology(LinkTopology.TWO_LINKS) > +class TestSingleCoreForwardPerf(TestSuite): > + """Single core forwarding performance test suite.""" > + > + config: Config > + > + def set_up_suite(self): > + """Set up the test suite.""" > + self.test_parameters = self.config.test_parameters > + self.delta_tolerance = self.config.delta_tolerance > + > + def _transmit(self, testpmd: TestPmd, frame_size: int) -> float: > + """Create a testpmd session with every rule in the given list, > verify jump behavior. > + > + Args: > + testpmd: The testpmd shell to use for forwarding packets. > + frame_size: The size of the frame to transmit. > + > + Returns: > + The MPPS (millions of packets per second) forwarded by the > SUT. > + """ > + # Build packet with dummy values, and account for the 14B and 20B > Ether and IP headers > + packet = ( > + Ether(src="52:00:00:00:00:00") > + / IP(src="1.2.3.4", dst="192.18.1.0") > + / Raw(load="x" * (frame_size - 14 - 20)) > + ) > + > + testpmd.start() > + > + # Transmit for 30 seconds. > + stats = assess_performance_by_packet(packet=packet, duration=30) > + > + rx_mpps = stats.rx_pps / 1_000_000 > + > + return rx_mpps > + > + def _produce_stats_table(self, test_parameters: list[dict[str, int | > float]]) -> None: > + """Display performance results in table format and write to > structured JSON file. > + > + Args: > + test_parameters: The expected and real stats per set of test > parameters. > + """ > + header = f"{'Frame Size':>12} | {'TXD/RXD':>12} | {'Real > MPPS':>12} | {'Expected MPPS':>14}" > + print("-" * len(header)) > + print(header) > + print("-" * len(header)) > + for params in test_parameters: > + print(f"{params['frame_size']:>12} | > {params['num_descriptors']:>12} | ", end="") > + print(f"{params['measured_mpps']:>12.2f} | > {params['expected_mpps']:>14.2f}") > + print("-" * len(header)) > + > + write_performance_json({"results": test_parameters}) > + > + @perf_test > + def single_core_forward_perf(self) -> None: > + """Validate expected single core forwarding performance. > + > + Steps: > + * Create a packet according to the frame size specified in > the test config. > + * Transmit from the traffic generator's ports 0 and 1 at > above the expect. > + * Forward on TestPMD's interfaces 0 and 1 with 1 core. > + > + Verify: > + * The resulting MPPS forwarded is greater than > expected_mpps*(1-delta_tolerance). > + """ > + # Find SUT DPDK driver to determine driver specific performance > optimization flags > + sut_dpdk_driver = > self._ctx.sut_node.config.ports[0].os_driver_for_dpdk > + > + for params in self.test_parameters: > + frame_size = params["frame_size"] > + num_descriptors = params["num_descriptors"] > + > + driver_specific_testpmd_args: TestPmdParamsDict = { > + "tx_ring": TXRingParams(descriptors=num_descriptors), > + "rx_ring": RXRingParams(descriptors=num_descriptors), > + "nb_cores": 1, > + } > + > + if sut_dpdk_driver == "mlx5_core": > + driver_specific_testpmd_args["burst"] = 64 > + driver_specific_testpmd_args["mbcache"] = 512 > + elif sut_dpdk_driver == "i40e": > + driver_specific_testpmd_args["rx_queues"] = 2 > + driver_specific_testpmd_args["tx_queues"] = 2 > + > + with TestPmd( > + **driver_specific_testpmd_args, > + ) as testpmd: > + params["measured_mpps"] = self._transmit(testpmd, > frame_size) > + params["performance_delta"] = ( > + float(params["measured_mpps"]) - > float(params["expected_mpps"]) > + ) / float(params["expected_mpps"]) > + params["pass"] = float(params["performance_delta"]) >= > -self.delta_tolerance > + > + self._produce_stats_table(self.test_parameters) > + > + for params in self.test_parameters: > + verify( > + params["pass"] is True, > + f"""Packets forwarded is less than {(1 > -self.delta_tolerance)*100}% > + of the expected baseline. > + Measured MPPS = {params["measured_mpps"]} > + Expected MPPS = {params["expected_mpps"]}""", > + ) > -- > 2.49.0 > >