Fix/reload report (#1062)

* fix reload report

* adjust impl & tests

* address review comments

* add newsfrag

* fix json parsing

* address review comment

doc/newsfragments/2906_changed.fix_interactive_reload_report.rst

@@ -0,0 +1 @@
+Fix a bug that causing interactive mode to render broken report after reloading source code.

testplan/common/report/base.py

@@ -439,6 +439,14 @@ def hash(self):
         """Return a hash of all entries in this report."""
         return hash((self.uid, tuple(id(entry) for entry in self.entries)))
 
+    def inherit(self, deceased: Self) -> Self:
+        """
+        Inherit certain information from the old report, mainly for information
+        preservation across interactive mode reloads.
+        """
+
+        raise NotImplementedError
+
 
 class BaseReportGroup(Report):
     """
@@ -578,6 +586,14 @@ def get_by_uids(self, uids):
             report = report.get_by_uid(uid)
         return report
 
+    def has_uid(self, uid):
+        """
+        Has a child report of `uid`
+        """
+        return uid in self._index
+
+    __contains__ = has_uid
+
     def get_by_uid(self, uid):
         """
         Get child report via `uid` lookup.
@@ -587,15 +603,7 @@ def get_by_uid(self, uid):
         """
         return self.entries[self._index[uid]]
 
-    def has_uid(self, uid):
-        """
-        Has a child report of `uid`
-        """
-        return uid in self._index
-
-    def __getitem__(self, uid):
-        """Shortcut to `get_by_uid()` method via [] operator."""
-        return self.get_by_uid(uid)
+    __getitem__ = get_by_uid
 
     def set_by_uid(self, uid, item):
         """
@@ -621,9 +629,13 @@ def set_by_uid(self, uid, item):
         else:
             self.append(item)
 
-    def __setitem__(self, uid, item):
-        """Shortcut to `set_by_uid()` method via [] operator."""
-        self.set_by_uid(uid, item)
+    __setitem__ = set_by_uid
+
+    def remove_by_uid(self, uid):
+        self.entries.pop(self._index[uid])
+        self._index = {child.uid: i for i, child in enumerate(self)}
+
+    __delitem__ = remove_by_uid
 
     @property
     def entry_uids(self):

testplan/report/testing/base.py

@@ -220,6 +220,23 @@ def filter(self, *functions, **kwargs) -> Self:
             result.propagate_tag_indices()
         return result
 
+    def inherit(self, deceased: Self) -> Self:
+        self.timer = deceased.timer
+        self.status_override = deceased.status_override
+        self.status_reason = deceased.status_reason
+        self.attachments = deceased.attachments
+        self.logs = deceased.logs
+
+        for uid in set(self.entry_uids) & set(deceased.entry_uids):
+            self_ = self[uid]
+            deceased_ = deceased[uid]
+            if isinstance(self_, TestGroupReport) and isinstance(
+                deceased_, TestGroupReport
+            ):
+                self_.inherit(deceased_)
+
+        return self
+
 
 class TestGroupReport(BaseReportGroup):
     """
@@ -419,6 +436,27 @@ def filter(self, *functions, **kwargs) -> Self:
             result.propagate_tag_indices()
         return result
 
+    def inherit(self, deceased: Self) -> Self:
+        self.timer = deceased.timer
+        self.status_override = deceased.status_override
+        self.status_reason = deceased.status_reason
+        self.env_status = deceased.env_status
+        self.logs = deceased.logs
+
+        for uid in set(self.entry_uids) & set(deceased.entry_uids):
+            self_ = self[uid]
+            deceased_ = deceased[uid]
+            if isinstance(self_, TestGroupReport) and isinstance(
+                deceased_, TestGroupReport
+            ):
+                self_.inherit(deceased_)
+            elif isinstance(self_, TestCaseReport) and isinstance(
+                deceased_, TestCaseReport
+            ):
+                self_.inherit(deceased_)
+
+        return self
+
 
 class TestCaseReport(Report):
     """
@@ -652,3 +690,14 @@ def pass_if_empty(self):
         """Mark as PASSED if this testcase contains no entries."""
         if not self.entries:
             self._status = Status.PASSED
+
+    def inherit(self, deceased: Self) -> Self:
+        self.timer = deceased.timer
+        self.runtime_status = deceased.runtime_status
+        self.status = deceased.status
+        self.status_override = deceased.status_override
+        self.status_reason = deceased.status_reason
+        self.attachments = deceased.attachments
+        self.logs = deceased.logs
+        self.entries = deceased.entries
+        return self

testplan/runnable/interactive/base.py

@@ -17,6 +17,7 @@
     RuntimeStatus,
     Status,
     TestGroupReport,
+    TestCaseReport,
     TestReport,
 )
 from testplan.runnable.interactive import http, reloader, resource_loader
@@ -740,30 +741,7 @@ def reload(self, rebuild_dependencies=False):
     def reload_report(self):
         """Update report with added/removed testcases"""
         new_report = self._initial_report()
-        for multitest in self.report.entries:  # multitest level
-            for suite_index, suite in enumerate(multitest.entries):
-                new_suite = new_report[multitest.uid][suite.uid]
-                for case_index, case in enumerate(suite.entries):
-                    try:
-                        if isinstance(case, TestGroupReport):
-                            for param_index, param_case in enumerate(
-                                case.entries
-                            ):
-                                try:
-                                    new_report[multitest.uid][suite.uid][
-                                        case.uid
-                                    ].entries[param_index] = case[
-                                        param_case.uid
-                                    ]
-                                except (KeyError, IndexError):
-                                    continue
-                        else:
-                            new_report[multitest.uid][suite.uid].entries[
-                                case_index
-                            ] = suite[case.uid]
-                    except (KeyError, IndexError):
-                        continue
-                multitest.entries[suite_index] = new_suite
+        self._report = new_report.inherit(self.report)
 
     def _setup_http_handler(self):
         """
@@ -804,7 +782,7 @@ def _display_connection_info(self):
             get_hostname_access_url(port, "/interactive"),
         )
 
-    def _initial_report(self):
+    def _initial_report(self) -> TestReport:
         """Generate the initial report skeleton."""
         report = TestReport(
             name=self.cfg.name,

testplan/web_ui/testing/src/Report/InteractiveReport.js

@@ -287,7 +287,7 @@ class InteractiveReportComponent extends BaseReport {
   putUpdatedReportEntry(updatedReportEntry) {
     const apiUrl = this.getApiUrl(updatedReportEntry);
     return axios
-      .put(apiUrl, updatedReportEntry)
+      .put(apiUrl, updatedReportEntry, { transformResponse: parseToJson })
       .then((response) => {
         if (response.data.errmsg) {
           console.error(response.data);

tests/functional/testplan/runnable/interactive/reload_cases/case_0_assertions.py

@@ -0,0 +1,32 @@
+from testplan.common.entity import ResourceStatus
+from testplan.common.report import RuntimeStatus, Status
+
+
+def prev_assertions(report):
+    assert report.entries[0].env_status == ResourceStatus.STARTED
+    suite_report = report.entries[0].entries[0]
+    assert suite_report.entries[0].name == "case_1"
+    assert suite_report.entries[0].runtime_status == RuntimeStatus.FINISHED
+    assert suite_report.entries[0].status == Status.PASSED
+    assert len(suite_report.entries[0].entries)
+    assert suite_report.entries[1].name == "case_3"
+    assert suite_report.entries[1].runtime_status == RuntimeStatus.FINISHED
+    assert suite_report.entries[1].status == Status.PASSED
+    assert len(suite_report.entries[1].entries)
+
+
+def curr_assertions(report):
+    assert report.entries[0].env_status == ResourceStatus.STARTED
+    suite_report = report.entries[0].entries[0]
+    assert suite_report.entries[0].name == "case_1"
+    assert suite_report.entries[0].runtime_status == RuntimeStatus.FINISHED
+    assert suite_report.entries[0].status == Status.PASSED
+    assert len(suite_report.entries[0].entries)
+    assert suite_report.entries[1].name == "case_2"
+    assert suite_report.entries[1].runtime_status == RuntimeStatus.READY
+    assert suite_report.entries[1].status == Status.UNKNOWN
+    assert not len(suite_report.entries[1].entries)
+    assert suite_report.entries[2].name == "case_3"
+    assert suite_report.entries[2].runtime_status == RuntimeStatus.FINISHED
+    assert suite_report.entries[2].status == Status.PASSED
+    assert len(suite_report.entries[2].entries)

tests/functional/testplan/runnable/interactive/reload_cases/case_0_curr.py

@@ -0,0 +1,22 @@
+from testplan.runners.pools.tasks.base import task_target
+from testplan.testing.multitest import MultiTest, testcase, testsuite
+
+
+@task_target
+def dummy_test():
+    return MultiTest(name="dummy_test", suites=[Suite()])
+
+
+@testsuite
+class Suite:
+    @testcase
+    def case_1(self, env, result):
+        result.true(True)
+
+    @testcase
+    def case_2(self, env, result):
+        result.false(False)
+
+    @testcase
+    def case_3(self, env, result):
+        result.true(True)

tests/functional/testplan/runnable/interactive/reload_cases/case_0_prev.py

@@ -0,0 +1,18 @@
+from testplan.runners.pools.tasks.base import task_target
+from testplan.testing.multitest import MultiTest, testcase, testsuite
+
+
+@task_target
+def dummy_test():
+    return MultiTest(name="dummy_test", suites=[Suite()])
+
+
+@testsuite
+class Suite:
+    @testcase
+    def case_1(self, env, result):
+        result.true(True)
+
+    @testcase
+    def case_3(self, env, result):
+        result.true(True)

tests/functional/testplan/runnable/interactive/reload_cases/case_1_assertions.py

@@ -0,0 +1,36 @@
+from testplan.common.entity import ResourceStatus
+from testplan.common.report import RuntimeStatus, Status
+
+
+def prev_assertions(report):
+    assert report.entries[0].env_status == ResourceStatus.STARTED
+    case_report = report.entries[0].entries[0].entries[0]
+    assert case_report.name == "case_1"
+    assert case_report.status == Status.FAILED
+    assert case_report.runtime_status == RuntimeStatus.FINISHED
+    assert case_report.entries[0].name == "case_1 <arg=0>"
+    assert case_report.entries[0].status == Status.PASSED
+    assert case_report.entries[0].runtime_status == RuntimeStatus.FINISHED
+    assert case_report.entries[1].name == "case_1 <arg=1>"
+    assert case_report.entries[1].status == Status.FAILED
+    assert case_report.entries[1].runtime_status == RuntimeStatus.FINISHED
+
+
+def curr_assertions(report):
+    assert report.entries[0].env_status == ResourceStatus.STARTED
+    case_report = report.entries[0].entries[0].entries[0]
+    assert case_report.name == "case_1"
+    assert case_report.status == Status.FAILED
+    assert case_report.runtime_status == RuntimeStatus.READY
+    assert case_report.entries[0].name == "case_1 <arg=0>"
+    assert case_report.entries[0].status == Status.PASSED
+    assert case_report.entries[0].runtime_status == RuntimeStatus.FINISHED
+    assert len(case_report.entries[0].entries)
+    assert case_report.entries[1].name == "case_1 <arg=1>"
+    assert case_report.entries[1].status == Status.FAILED
+    assert case_report.entries[1].runtime_status == RuntimeStatus.FINISHED
+    assert len(case_report.entries[1].entries)
+    assert case_report.entries[2].name == "case_1 <arg=2>"
+    assert case_report.entries[2].status == Status.UNKNOWN
+    assert case_report.entries[2].runtime_status == RuntimeStatus.READY
+    assert not len(case_report.entries[2].entries)

tests/functional/testplan/runnable/interactive/reload_cases/case_1_curr.py

@@ -0,0 +1,14 @@
+from testplan.runners.pools.tasks.base import task_target
+from testplan.testing.multitest import MultiTest, testcase, testsuite
+
+
+@task_target
+def dummy_test():
+    return MultiTest(name="dummy_test", suites=[Suite()])
+
+
+@testsuite
+class Suite:
+    @testcase(parameters=range(3))
+    def case_1(self, env, result, arg):
+        result.true(True)

tests/functional/testplan/runnable/interactive/reload_cases/case_1_prev.py

@@ -0,0 +1,17 @@
+from testplan.runners.pools.tasks.base import task_target
+from testplan.testing.multitest import MultiTest, testcase, testsuite
+
+
+@task_target
+def dummy_test():
+    return MultiTest(name="dummy_test", suites=[Suite()])
+
+
+@testsuite
+class Suite:
+    @testcase(parameters=range(2))
+    def case_1(self, env, result, arg):
+        if arg == 0:
+            result.true(True)
+        else:
+            result.true(False)

tests/functional/testplan/runnable/interactive/reload_cases/case_2_assertions.py

@@ -0,0 +1,37 @@
+from testplan.common.entity import ResourceStatus
+from testplan.common.report import RuntimeStatus, Status
+
+
+def prev_assertions(report):
+    assert report.entries[0].env_status == ResourceStatus.STARTED
+    suite_report = report.entries[0].entries[0]
+    assert suite_report.entries[0].name == "case_1"
+    assert suite_report.entries[0].status == Status.FAILED
+    assert suite_report.entries[0].runtime_status == RuntimeStatus.FINISHED
+    assert suite_report.entries[1].name == "case_2"
+    assert suite_report.entries[1].status == Status.PASSED
+    assert suite_report.entries[1].runtime_status == RuntimeStatus.FINISHED
+    assert suite_report.entries[2].name == "case_3"
+    assert suite_report.entries[2].status == Status.PASSED
+    assert suite_report.entries[2].runtime_status == RuntimeStatus.FINISHED
+
+
+def curr_assertions(report):
+    assert report.entries[0].env_status == ResourceStatus.STARTED
+    suite_report = report.entries[0].entries[0]
+    assert suite_report.entries[0].name == "case_1"
+    assert suite_report.entries[0].status == Status.UNKNOWN
+    assert suite_report.entries[0].runtime_status == RuntimeStatus.READY
+    assert not len(suite_report.entries[0].entries)
+    assert suite_report.entries[1].entries[0].name == "case_3 <arg=0>"
+    assert suite_report.entries[1].entries[0].status == Status.UNKNOWN
+    assert (
+        suite_report.entries[1].entries[0].runtime_status
+        == RuntimeStatus.READY
+    )
+    assert suite_report.entries[1].entries[1].name == "case_3 <arg=1>"
+    assert suite_report.entries[1].entries[1].status == Status.UNKNOWN
+    assert (
+        suite_report.entries[1].entries[1].runtime_status
+        == RuntimeStatus.READY
+    )

tests/functional/testplan/runnable/interactive/reload_cases/case_2_curr.py

@@ -0,0 +1,18 @@
+from testplan.runners.pools.tasks.base import task_target
+from testplan.testing.multitest import MultiTest, testcase, testsuite
+
+
+@task_target
+def dummy_test():
+    return MultiTest(name="dummy_test", suites=[Suite()])
+
+
+@testsuite
+class Suite:
+    @testcase
+    def case_1(self, env, result):
+        result.true(True)
+
+    @testcase(parameters=range(2))
+    def case_3(self, env, result, arg):
+        result.true(True)