fix: Raise exceptions for DUTs created by DutFactory (RDT-1522)

pytest-embedded-idf/tests/test_idf.py

@@ -101,7 +101,7 @@ def test_idf_run_all_single_board_cases():
         '--junitxml',
         'report.xml',
     )
-    result.assert_outcomes(passed=4, errors=0)  # FIXME, dut-factory mode can't raise error now
+    result.assert_outcomes(passed=3, failed=1)
 
     junit_report = ET.parse('report.xml').getroot()[0]
 

pytest-embedded/pytest_embedded/dut_factory.py

@@ -838,3 +838,16 @@ def create(
                 _close_or_terminate(obj)
             del layout
             raise e
+
+    @classmethod
+    def get_all_duts(cls) -> list[Dut]:
+        """Get all DUTs created by DutFactory."""
+
+        duts = []
+        for layout in cls.obj_stack:
+            # The DUT is always the last object in the layout
+            dut = layout[-1]
+            if isinstance(dut, Dut):
+                duts.append(dut)
+
+        return duts

pytest-embedded/pytest_embedded/plugin.py

@@ -1372,10 +1372,18 @@ def pytest_collection_modifyitems(self, config: Config, items: list[Function]):
 
     @pytest.hookimpl(trylast=True)
     def pytest_runtest_call(self, item: Function):
-        # raise dut failed cases
+        all_duts: list[Dut] = []
+
+        # Check DUTs created by fixture
         if 'dut' in item.funcargs:
-            duts = [dut for dut in to_list(item.funcargs['dut']) if isinstance(dut, Dut)]
-            self._raise_dut_failed_cases_if_exists(duts)  # type: ignore
+            fixture_duts = [dut for dut in to_list(item.funcargs['dut']) if isinstance(dut, Dut)]
+            all_duts.extend(fixture_duts)
+
+        # Check DUTs created by DutFactory
+        factory_duts = DutFactory.get_all_duts()
+        all_duts.extend(factory_duts)
+
+        self._raise_dut_failed_cases_if_exists(all_duts)  # type: ignore
 
     @pytest.hookimpl(trylast=True)  # combine all possible junit reports should be the last step
     def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None: