Have chip-repl yamltests parser use PICS to determine if test step sh…

…ould run (#24626)

* Have chip-repl yamltests parser use PICs

* Fix small issue

* Remove flaky tests from chip-repl yamltests runner for now

scripts/tests/chiptest/__init__.py

@@ -95,7 +95,6 @@ def _hardcoded_python_yaml_tests():
         "Test_TC_I_2_1.yaml",
         "Test_TC_ILL_1_1.yaml",
         "Test_TC_ILL_2_1.yaml",
-        "Test_TC_LVL_2_1.yaml",
         "Test_TC_LVL_2_2.yaml",
         "Test_TC_LCFG_1_1.yaml",
         "Test_TC_LTIME_1_2.yaml",
@@ -166,7 +165,6 @@ def _hardcoded_python_yaml_tests():
         "Test_TC_WNCV_2_3.yaml",
         "Test_TC_WNCV_4_3.yaml",
         "Test_TC_WNCV_4_4.yaml",
-        "DL_Schedules.yaml",
         "DL_UsersAndCredentials.yaml",
     ]
 

scripts/tests/chiptest/test_definition.py

@@ -283,7 +283,7 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str, ti
             elif self.use_chip_repl_yaml_tester:
                 chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd
                 python_cmd = chip_repl_yaml_tester_cmd + \
-                    ['--setup-code', app.setupCode] + ['--yaml-path', self.run_name]
+                    ['--setup-code', app.setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file]
                 runner.RunSubprocess(python_cmd, name='CHIP_REPL_YAML_TESTER',
                                      dependencies=[apps_register], timeout_seconds=timeout_seconds)
             else:

scripts/tests/chiptest/yamltest_with_chip_repl_tester.py

@@ -52,7 +52,11 @@
     '--node-id',
     default=0x12344321,
     help='Node ID to use when commissioning device')
-def main(setup_code, yaml_path, node_id):
+@click.option(
+    '--pics-file',
+    default=None,
+    help='Optional PICS file')
+def main(setup_code, yaml_path, node_id, pics_file):
     # Setting up python environment for running YAML CI tests using python parser.
     with tempfile.NamedTemporaryFile() as chip_stack_storage:
         chip.native.Init()
@@ -79,21 +83,23 @@ def main(setup_code, yaml_path, node_id):
             clusters_definitions = SpecDefinitionsFromPath(_CLUSTER_XML_DIRECTORY_PATH + '/*/*.xml')
 
             # Parsing YAML test and setting up chip-repl yamltests runner.
-            yaml = TestParser(yaml_path, None, clusters_definitions)
+            yaml = TestParser(yaml_path, pics_file, clusters_definitions)
             runner = ReplTestRunner(clusters_definitions, certificate_authority_manager, dev_ctrl)
 
             # Executing and validating test
             for test_step in yaml.tests:
                 test_action = runner.encode(test_step)
                 # TODO if test_action is None we should see if it is a pseudo cluster.
-                if test_action is not None:
-                    response = runner.execute(test_action)
-                    decoded_response = runner.decode(response)
-                    post_processing_result = test_step.post_process_response(decoded_response)
-                    if not post_processing_result.is_success():
-                        raise Exception(f'Test step failed {test_step.label}')
-                else:
+                if test_action is None:
                     raise Exception(f'Failed to encode test step {test_step.label}')
+                if not test_action.pics_enabled:
+                    continue
+
+                response = runner.execute(test_action)
+                decoded_response = runner.decode(response)
+                post_processing_result = test_step.post_process_response(decoded_response)
+                if not post_processing_result.is_success():
+                    raise Exception(f'Test step failed {test_step.label}')
         except Exception:
             print(traceback.format_exc())
             exit(-2)

src/controller/python/chip/yaml/runner.py

@@ -74,9 +74,10 @@ class _ExecutionContext:
 class BaseAction(ABC):
     '''Interface for a single YAML action that is to be executed.'''
 
-    def __init__(self, label, identity):
-        self._label = label
-        self._identity = identity
+    def __init__(self, test_step):
+        self._label = test_step.label
+        self._identity = test_step.identity
+        self._pics_enabled = test_step.is_pics_enabled
 
     @property
     def label(self):
@@ -86,6 +87,10 @@ def label(self):
     def identity(self):
         return self._identity
 
+    @property
+    def pics_enabled(self):
+        return self._pics_enabled
+
     @abstractmethod
     def run_action(self, dev_ctrl: ChipDeviceCtrl) -> _ActionResult:
         pass
@@ -106,7 +111,7 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext):
             action to perform for this write attribute.
           UnexpectedParsingError: Raised if there is an unexpected parsing error.
         '''
-        super().__init__(test_step.label, test_step.identity)
+        super().__init__(test_step)
         self._busy_wait_ms = test_step.busy_wait_ms
         self._command_name = stringcase.pascalcase(test_step.command)
         self._cluster = cluster
@@ -167,7 +172,7 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext):
             action to perform for this read attribute.
           UnexpectedParsingError: Raised if there is an unexpected parsing error.
         '''
-        super().__init__(test_step.label, test_step.identity)
+        super().__init__(test_step)
         self._attribute_name = stringcase.pascalcase(test_step.attribute)
         self._cluster = cluster
         self._endpoint = test_step.endpoint
@@ -235,7 +240,7 @@ class WaitForCommissioneeAction(BaseAction):
     ''' Wait for commissionee action to be executed.'''
 
     def __init__(self, test_step):
-        super().__init__(test_step.label, test_step.identity)
+        super().__init__(test_step)
         self._node_id = test_step.node_id
         self._expire_existing_session = False
         # This is the default when no timeout is provided.
@@ -357,7 +362,7 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext):
             action to perform for this write attribute.
           UnexpectedParsingError: Raised if there is an unexpected parsing error.
         '''
-        super().__init__(test_step.label, test_step.identity)
+        super().__init__(test_step)
         self._attribute_name = stringcase.pascalcase(test_step.attribute)
         self._busy_wait_ms = test_step.busy_wait_ms
         self._cluster = cluster
@@ -421,7 +426,7 @@ def __init__(self, test_step, context: _ExecutionContext):
         Raises:
           UnexpectedParsingError: Raised if the expected queue does not exist.
         '''
-        super().__init__(test_step.label, test_step.identity)
+        super().__init__(test_step)
         self._attribute_name = stringcase.pascalcase(test_step.attribute)
         self._output_queue = context.subscription_callback_result_queue.get(self._attribute_name,
                                                                             None)
@@ -451,7 +456,7 @@ def __init__(self, test_step):
         Raises:
           UnexpectedParsingError: Raised if the expected queue does not exist.
         '''
-        super().__init__(test_step.label, test_step.identity)
+        super().__init__(test_step)
         if test_step.command != 'PairWithCode':
             raise UnexpectedParsingError(f'Unexpected CommisionerCommand {test_step.command}')