[PATCH v2 3/3] dts: add flow validation

[Dean Marx <dmarx@iol.unh.edu>]

Add a method for validating flow rules to the testpmd shell class.
Implement test case skipping for flow rules that do not pass
validation.

Signed-off-by: Dean Marx <dmarx@iol.unh.edu>
---
 dts/framework/remote_session/testpmd_shell.py | 15 ++++
 dts/framework/test_run.py                     |  3 +
 dts/framework/test_suite.py                   | 21 ++++-
 dts/tests/TestSuite_flow.py                   | 90 +++++++++++++------
 4 files changed, 102 insertions(+), 27 deletions(-)

diff --git a/dts/framework/remote_session/testpmd_shell.py b/dts/framework/remote_session/testpmd_shell.py
index bd5f2659bd..c2c7ade60a 100644
--- a/dts/framework/remote_session/testpmd_shell.py
+++ b/dts/framework/remote_session/testpmd_shell.py
@@ -1961,6 +1961,21 @@ def flow_create(self, flow_rule: FlowRule, port_id: int) -> int:
             self._logger.debug(f"Failed to create flow rule:\n{flow_output}")
             raise InteractiveCommandExecutionError(f"Failed to create flow rule:\n{flow_output}")
 
+    def flow_validate(self, flow_rule: FlowRule, port_id: int) -> bool:
+        """Validates a flow rule in the testpmd session.
+
+        Args:
+            flow_rule: :class:`FlowRule` object used for validating testpmd flow rule.
+            port_id: Integer representing the port to use.
+
+        Returns:
+            Boolean representing whether rule is valid or not.
+        """
+        flow_output = self.send_command(f"flow validate {port_id} {flow_rule}")
+        if "Flow rule validated" in flow_output:
+            return True
+        return False
+
     def flow_delete(self, flow_id: int, port_id: int, verify: bool = True) -> None:
         """Deletes the specified flow rule from the testpmd session.
 
diff --git a/dts/framework/test_run.py b/dts/framework/test_run.py
index 0fdc57ea9c..cf4af19e8f 100644
--- a/dts/framework/test_run.py
+++ b/dts/framework/test_run.py
@@ -644,6 +644,9 @@ def next(self) -> State | None:
                 return self
 
             self.result.update(Result.FAIL, e)
+        except SkippedTestException as e:
+            self.logger.info(f"Test case execution SKIPPED: {e}")
+            self.result.update(Result.SKIP, e)
         else:
             self.result.update(Result.PASS)
             self.logger.info(f"{self.description.capitalize()} PASSED.")
diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py
index e07c327b77..34e9eb54ea 100644
--- a/dts/framework/test_suite.py
+++ b/dts/framework/test_suite.py
@@ -39,7 +39,7 @@
     PacketFilteringConfig,
 )
 
-from .exception import ConfigurationError, InternalError, TestCaseVerifyError
+from .exception import ConfigurationError, InternalError, SkippedTestException, TestCaseVerifyError
 from .logger import DTSLogger, get_dts_logger
 from .utils import get_packet_summaries, to_pascal_case
 
@@ -411,6 +411,25 @@ def _fail_test_case_verify(self, failure_description: str) -> None:
             self._logger.debug(command_res.command)
         raise TestCaseVerifyError(failure_description)
 
+    def verify_skip(self, condition: bool, skip_reason: str) -> None:
+        """Verify `condition` and handle skips.
+
+        When `condition` is :data:`False`, raise a skip exception.
+
+        Args:
+            condition: The condition to check.
+            skip_reason: Description of the reason for skipping.
+
+        Raises:
+            SkippedTestException: `condition` is :data:`False`.
+        """
+        if not condition:
+            self._skip_test_case_verify(skip_reason)
+
+    def _skip_test_case_verify(self, skip_description: str) -> None:
+        self._logger.debug(f"Test case skipped: {skip_description}")
+        raise SkippedTestException(skip_description)
+
     def verify_packets(self, expected_packet: Packet, received_packets: list[Packet]) -> None:
         """Verify that `expected_packet` has been received.
 
diff --git a/dts/tests/TestSuite_flow.py b/dts/tests/TestSuite_flow.py
index 06bd3bedc5..6c5f9b0e8e 100644
--- a/dts/tests/TestSuite_flow.py
+++ b/dts/tests/TestSuite_flow.py
@@ -19,6 +19,7 @@
 from scapy.layers.l2 import Dot1Q, Ether
 from scapy.packet import Packet, Raw
 
+from framework.exception import InteractiveCommandExecutionError
 from framework.remote_session.testpmd_shell import FlowRule, TestPmdShell
 from framework.test_suite import TestSuite, func_test
 from framework.testbed_model.capability import NicCapability, TopologyType, requires
@@ -84,10 +85,17 @@ def zip_lists(
 
         with TestPmdShell(rx_queues=4, tx_queues=4) as testpmd:
             for flow, packet, expected_packet in zip_lists(flows, packets, expected_packets):
-                flow_id = testpmd.flow_create(flow_rule=flow, port_id=0)
+                is_valid = testpmd.flow_validate(flow_rule=flow, port_id=0)
+                self.verify_skip(is_valid, "flow rule failed validation.")
+
+                try:
+                    flow_id = testpmd.flow_create(flow_rule=flow, port_id=0)
+                except InteractiveCommandExecutionError:
+                    self.log("Flow rule validation passed, but flow creation failed.")
+                    self.verify(False, "Failed flow creation")
 
                 if verification_method == self.send_packet_and_verify:
-                    verification_method(*args, **kwargs)
+                    verification_method(packet=packet, *args, **kwargs)
 
                 elif verification_method == self.send_packet_and_verify_queue:
                     verification_method(
@@ -125,7 +133,7 @@ def send_packet_and_verify_queue(
             test_queue: Represents the queue the test packet is being sent to.
             testpmd: TestPmdShell instance being used to send test packet.
         """
-        testpmd.set_verbose(level=1)
+        testpmd.set_verbose(level=8)
         testpmd.start()
         self.send_packet_and_capture(packet=packet)
         verbose_output = testpmd.extract_verbose_output(testpmd.stop())
@@ -186,9 +194,16 @@ def send_packet_and_verify_jump(
             test_queues: List of Rx queue IDs each packet should be received on.
             testpmd: TestPmdShell instance to create flows on.
         """
-        testpmd.set_verbose(level=1)
+        testpmd.set_verbose(level=8)
         for flow in flow_rules:
-            testpmd.flow_create(flow_rule=flow, port_id=0)
+            is_valid = testpmd.flow_validate(flow_rule=flow, port_id=0)
+            self.verify_skip(is_valid, "flow rule failed validation.")
+
+            try:
+                testpmd.flow_create(flow_rule=flow, port_id=0)
+            except InteractiveCommandExecutionError:
+                self.log("Flow validation passed, but flow creation failed.")
+                self.verify(False, "Failed flow creation")
 
         for packet, test_queue in zip(packets, test_queues):
             testpmd.start()
@@ -397,12 +412,9 @@ def test_drop_action_ETH(self) -> None:
         received under normal circumstances.
         """
         packet_list = [
-            Ether(src="02:00:00:00:00:00"),
-            Raw(load="xxxxx"),
-            Ether(dst="02:00:00:00:00:00"),
-            Raw(load="xxxxx"),
-            Ether(type=0x0800),
-            Raw(load="xxxxx"),
+            Ether(src="02:00:00:00:00:00") / Raw(load="xxxxx"),
+            Ether(dst="02:00:00:00:00:00") / Raw(load="xxxxx"),
+            Ether(type=0x0800) / Raw(load="xxxxx"),
         ]
         flow_list = [
             FlowRule(
@@ -413,8 +425,12 @@ def test_drop_action_ETH(self) -> None:
             ),
             FlowRule(direction="ingress", pattern=["eth type is 0x0800"], actions=["drop"]),
         ]
-        # Verify packet reception without flow rule
-        self.send_packet_and_verify(packet=Raw(load="xxxxx"), should_receive=True)
+        # verify reception with test packet
+        packet = Ether() / IP() / Raw(load="xxxxx")
+        with TestPmdShell() as testpmd:
+            testpmd.start()
+            received = self.send_packet_and_capture(packet)
+            self.verify(received != [], "Test packet was never received.")
         self.runner(
             verification_method=self.send_packet_and_verify,
             flows=flow_list,
@@ -463,8 +479,12 @@ def test_drop_action_IP(self) -> None:
             ),
             FlowRule(direction="ingress", pattern=["eth / ipv6 proto is 17"], actions=["drop"]),
         ]
-        # Verify packet reception without flow rule
-        self.send_packet_and_verify(packet=Raw(load="xxxxx"), should_receive=True)
+        # verify reception with test packet
+        packet = Ether() / IP() / Raw(load="xxxxx")
+        with TestPmdShell() as testpmd:
+            testpmd.start()
+            received = self.send_packet_and_capture(packet)
+            self.verify(received != [], "Test packet was never received.")
         self.runner(
             verification_method=self.send_packet_and_verify,
             flows=flow_list,
@@ -509,8 +529,12 @@ def test_drop_action_L4(self) -> None:
             ),
             FlowRule(direction="ingress", pattern=["eth / ipv4 / udp dst is 53"], actions=["drop"]),
         ]
-        # Verify packet reception without flow rule
-        self.send_packet_and_verify(packet=Raw(load="xxxxx"), should_receive=True)
+        # verify reception with test packet
+        packet = Ether() / IP() / Raw(load="xxxxx")
+        with TestPmdShell() as testpmd:
+            testpmd.start()
+            received = self.send_packet_and_capture(packet)
+            self.verify(received != [], "Test packet was never received.")
         self.runner(
             verification_method=self.send_packet_and_verify,
             flows=flow_list,
@@ -543,8 +567,12 @@ def test_drop_action_VLAN(self) -> None:
             FlowRule(direction="ingress", pattern=["eth / vlan"], actions=["drop"]),
             FlowRule(direction="ingress", pattern=["eth / vlan"], actions=["drop"]),
         ]
-        # Verify packet reception without flow rule
-        self.send_packet_and_verify(packet=Raw(load="xxxxx"), should_receive=True)
+        # verify reception with test packet
+        packet = Ether() / IP() / Raw(load="xxxxx")
+        with TestPmdShell() as testpmd:
+            testpmd.start()
+            received = self.send_packet_and_capture(packet)
+            self.verify(received != [], "Test packet was never received.")
         self.runner(
             verification_method=self.send_packet_and_verify,
             flows=flow_list,
@@ -615,9 +643,9 @@ def test_egress_rules(self) -> None:
         """
         packet_list = [
             Ether(src="02:00:00:00:00:00"),
-            IP(src="192.168.1.1"),
-            TCP(sport=1234),
-            UDP(sport=5000),
+            Ether() / IP(src="192.168.1.1"),
+            IP() / TCP(sport=1234),
+            IP() / UDP(sport=5000),
         ]
         flow_list = [
             FlowRule(
@@ -627,8 +655,12 @@ def test_egress_rules(self) -> None:
             FlowRule(direction="egress", pattern=["tcp src is 1234"], actions=["drop"]),
             FlowRule(direction="egress", pattern=["udp src is 5000"], actions=["drop"]),
         ]
-        # Verify packet reception without flow rule
-        self.send_packet_and_verify(packet=Raw(load="xxxxx"), should_receive=True)
+        # verify reception with test packet
+        packet = Ether() / IP() / Raw(load="xxxxx")
+        with TestPmdShell() as testpmd:
+            testpmd.start()
+            received = self.send_packet_and_capture(packet)
+            self.verify(received != [], "Test packet was never received.")
         self.runner(
             verification_method=self.send_packet_and_verify,
             flows=flow_list,
@@ -715,9 +747,15 @@ def test_priority_attribute(self) -> None:
         ]
         expected_queue_list = [1, 2, 3]
         with TestPmdShell(rx_queues=4, tx_queues=4) as testpmd:
-            testpmd.set_verbose(level=1)
+            testpmd.set_verbose(level=8)
             for flow, expected_queue in zip(flow_list, expected_queue_list):
-                testpmd.flow_create(flow_rule=flow, port_id=0)
+                is_valid = testpmd.flow_validate(flow_rule=flow, port_id=0)
+                self.verify_skip(is_valid, "flow rule failed validation.")
+                try:
+                    testpmd.flow_create(flow_rule=flow, port_id=0)
+                except InteractiveCommandExecutionError:
+                    self.log("Flow rule validation passed, but flow creation failed.")
+                    self.verify(False, "Failed flow creation")
                 testpmd.start()
                 self.send_packet_and_capture(test_packet)
                 verbose_output = testpmd.extract_verbose_output(testpmd.stop())
-- 
2.49.0