Skip to content

Commit

Permalink
Fix/certain assertions in wrong place in split-exported JSON (#1085)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhenyu-ms
zhenyu-ms committed May 23, 2024
1 parent 5705bfa commit fe16fa4
Show file tree
Hide file tree
Showing 7 changed files with 100 additions and 199 deletions.
4 changes: 1 addition & 3 deletions testplan/common/report/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -931,9 +931,7 @@ def set_runtime_status_filtered(
new_status, entries[entry.name]
)
else:
entry.set_runtime_status_filtered(
new_status, entries[entry.name]
)
entry.runtime_status = new_status
self._runtime_status = new_status

def _get_comparison_attrs(self):
Expand Down
7 changes: 3 additions & 4 deletions testplan/exporters/testing/json/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,7 @@
)
from testplan.common.utils.path import makedirs
from testplan.defaults import ATTACHMENTS, RESOURCE_DATA
from testplan.report import ReportCategories
from testplan.report.testing.base import TestReport
from testplan.report.testing.base import TestReport, TestCaseReport
from testplan.report.testing.schemas import TestReportSchema
from ..base import Exporter

Expand Down Expand Up @@ -216,7 +215,7 @@ def split_json_report(data):
def split_assertions(entries, assertions):
"""Remove assertions from report and place them in a dictionary."""
for entry in entries:
if entry.get("category") == ReportCategories.TESTCASE:
if entry["type"] == TestCaseReport.__name__:
assertions[entry["uid"]] = entry["entries"]
entry["entries"] = []
elif "entries" in entry:
Expand All @@ -234,7 +233,7 @@ def merge_json_report(meta, structure, assertions, strict=True):
def merge_assertions(entries, assertions, strict=True):
"""Fill assertions into report by the unique id."""
for entry in entries:
if entry.get("category") == ReportCategories.TESTCASE:
if entry["type"] == TestCaseReport.__name__:
if entry["uid"] in assertions:
entry["entries"] = assertions[entry["uid"]]
elif strict:
Expand Down
6 changes: 5 additions & 1 deletion testplan/runnable/interactive/http.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -952,7 +952,11 @@ def _extract_entries(entry: Dict) -> Dict:
"""
entries = {}

if entry["category"] == ReportCategories.TESTCASE:
# FIXME:
# by design, filtering out assertions has been properly done in frontend
# code; following lines are still kept for possible corner cases, while
# currently ``type`` field does not exist in ``Shallow*`` schemas...
if entry.get("type") == TestCaseReport.__name__:
return entries

for child in entry.get("entries", []):
Expand Down
16 changes: 10 additions & 6 deletions testplan/web_ui/testing/src/Report/InteractiveReport.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { Redirect } from "react-router-dom";
import { generatePath } from "react-router";
import base64url from "base64url";
import { atom, useAtomValue } from "jotai";
import _ from "lodash";

import BaseReport from "./BaseReport";
import Toolbar from "../Toolbar/Toolbar.js";
Expand Down Expand Up @@ -487,12 +488,10 @@ class InteractiveReportComponent extends BaseReport {
category: category,
};

if (entries) {
if (entries.length && !isReportLeaf(reportEntry)) {
pruneEntry.entries = entries.map((entry) =>
this.pruneReportEntry(entry)
);
}
if (!isReportLeaf(reportEntry) && !_.isEmpty(entries)) {
pruneEntry.entries = entries.map((entry) =>
this.pruneReportEntry(entry)
);
}

return pruneEntry;
Expand All @@ -505,6 +504,11 @@ class InteractiveReportComponent extends BaseReport {
*/
shallowReportEntry(reportEntry) {
const { entries, ...shallowEntry } = reportEntry;

if (isReportLeaf(reportEntry)) {
return shallowEntry;
}

shallowEntry.entry_uids = entries.map((entry) => entry.uid);

// the filter text is either "null" or an empty string, use truthy-falsy
Expand Down
2 changes: 1 addition & 1 deletion testplan/web_ui/testing/src/Report/reportUtils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ const getAssertions = (selectedEntries, displayTime, UTCTime) => {
return "";
}
timestamp = timestamp.split("+");
let label = UTCTime ? "Z" : timestamp.length == 2 ? "+" + timestamp[1] : "";
let label = UTCTime ? "Z" : timestamp.length === 2 ? "+" + timestamp[1] : "";
return format(new Date(timestamp[0]), "HH:mm:ss.SSS") + label;
};

Expand Down
244 changes: 61 additions & 183 deletions tests/functional/testplan/exporters/testing/test_json.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,179 +12,17 @@
from testplan.testing import multitest


full_report = {
"python_version": "3.7.5",
"category": "testplan",
"runtime_status": "finished",
"status": "failed",
"entries": [
{
"category": "multitest",
"parent_uids": ["Multiply"],
"name": "MultiplyTest",
"uid": "MultiplyTest",
"entries": [
{
"category": "testsuite",
"parent_uids": ["Multiply", "MultiplyTest"],
"name": "BasicSuite",
"uid": "BasicSuite",
"entries": [
{
"category": "parametrization",
"parent_uids": [
"Multiply",
"MultiplyTest",
"BasicSuite",
],
"name": "Basic Multiply",
"uid": "basic_multiply",
"entries": [
{
"entries": [
{
"name": "test assertion1",
"uid": "test_assertion1",
},
],
"type": "TestCaseReport",
"category": "testcase",
"parent_uids": [
"Multiply",
"MultiplyTest",
"BasicSuite",
"basic_multiply",
],
"name": "basic multiply <p1='aaa', p2=111>",
"uid": "basic_multiply__p1_aaa__p2_111",
},
{
"entries": [
{
"name": "test assertion2",
"uid": "test_assertion2",
}
],
"type": "TestCaseReport",
"category": "testcase",
"parent_uids": [
"Multiply",
"MultiplyTest",
"BasicSuite",
"basic_multiply",
],
"name": "basic multiply <p1='bbb', p2=222>",
"uid": "basic_multiply__p1_bbb__p2_222",
},
],
"type": "TestGroupReport",
}
],
"type": "TestGroupReport",
}
],
"type": "TestGroupReport",
}
],
"name": "Multiply",
"uid": "Multiply",
"project": "testplan",
"timeout": 14400,
}


meta_part = {
"python_version": "3.7.5",
"category": "testplan",
"runtime_status": "finished",
"status": "failed",
"entries": [],
"name": "Multiply",
"uid": "Multiply",
"project": "testplan",
"timeout": 14400,
}


structure_part = [
{
"category": "multitest",
"parent_uids": ["Multiply"],
"name": "MultiplyTest",
"uid": "MultiplyTest",
"entries": [
{
"category": "testsuite",
"parent_uids": ["Multiply", "MultiplyTest"],
"name": "BasicSuite",
"uid": "BasicSuite",
"entries": [
{
"category": "parametrization",
"parent_uids": [
"Multiply",
"MultiplyTest",
"BasicSuite",
],
"name": "Basic Multiply",
"uid": "basic_multiply",
"entries": [
{
"entries": [],
"type": "TestCaseReport",
"category": "testcase",
"parent_uids": [
"Multiply",
"MultiplyTest",
"BasicSuite",
"basic_multiply",
],
"name": "basic multiply <p1='aaa', p2=111>",
"uid": "basic_multiply__p1_aaa__p2_111",
},
{
"entries": [],
"type": "TestCaseReport",
"category": "testcase",
"parent_uids": [
"Multiply",
"MultiplyTest",
"BasicSuite",
"basic_multiply",
],
"name": "basic multiply <p1='bbb', p2=222>",
"uid": "basic_multiply__p1_bbb__p2_222",
},
],
"type": "TestGroupReport",
}
],
"type": "TestGroupReport",
}
],
"type": "TestGroupReport",
}
]


assertions_part = {
"basic_multiply__p1_aaa__p2_111": [
{"name": "test assertion1", "uid": "test_assertion1"},
],
"basic_multiply__p1_bbb__p2_222": [
{"name": "test assertion2", "uid": "test_assertion2"},
],
}


@multitest.testsuite
class Alpha:
def setup(self, env, result):
result.log("within suite setup...")

@multitest.testcase
def test_comparison(self, env, result):
result.equal(1, 1, "equality description")

@multitest.testcase
def test_membership(self, env, result):
@multitest.testcase(parameters=(1, 2, 3))
def test_membership(self, env, result, arg):
result.contain(1, [1, 2, 3])

@multitest.testcase
Expand All @@ -208,19 +46,40 @@ def test_error(self, env, result):
raise Exception("foo")


def test_split_and_merge():
def secondary_after_start(env, result):
result.log("within after start...")


def test_split_and_merge(runpath):
"""
Test static methods used for splitting and merging JSON report.
"""
json_path = os.path.join(runpath, "report.json")
plan = TestplanMock(
"plan", exporters=JSONExporter(json_path=json_path), runpath=runpath
)
multitest_1 = multitest.MultiTest(name="Primary", suites=[Alpha()])
multitest_2 = multitest.MultiTest(
name="Secondary", suites=[Beta()], after_start=secondary_after_start
)
plan.add(multitest_1)
plan.add(multitest_2)
plan.run()

assert os.path.exists(json_path)
assert os.stat(json_path).st_size > 0

with open(json_path) as json_file:
report = json.load(json_file)
del report["version"]

meta, structure, assertions = JSONExporter.split_json_report(
copy.deepcopy(full_report)
copy.deepcopy(report)
)
assert meta == meta_part
assert structure == structure_part
assert assertions == assertions_part
assert "information" in meta
assert meta["entries"] == []
assert (
JSONExporter.merge_json_report(meta, structure, assertions)
== full_report
JSONExporter.merge_json_report(meta, structure, assertions) == report
)


Expand All @@ -234,7 +93,9 @@ def test_json_exporter(runpath):
"plan", exporters=JSONExporter(json_path=json_path), runpath=runpath
)
multitest_1 = multitest.MultiTest(name="Primary", suites=[Alpha()])
multitest_2 = multitest.MultiTest(name="Secondary", suites=[Beta()])
multitest_2 = multitest.MultiTest(
name="Secondary", suites=[Beta()], after_start=secondary_after_start
)
plan.add(multitest_1)
plan.add(multitest_2)
plan.run()
Expand Down Expand Up @@ -273,7 +134,9 @@ def test_json_exporter_generating_split_report(runpath):
)

multitest_1 = multitest.MultiTest(name="Primary", suites=[Alpha()])
multitest_2 = multitest.MultiTest(name="Secondary", suites=[Beta()])
multitest_2 = multitest.MultiTest(
name="Secondary", suites=[Beta()], after_start=secondary_after_start
)
plan.add(multitest_1)
plan.add(multitest_2)
plan.run()
Expand Down Expand Up @@ -314,22 +177,37 @@ def test_json_exporter_generating_split_report(runpath):
assert structure[0]["name"] == "Primary" # 1st multitest name
assert len(structure[0]["entries"]) == 1 # one suite in 1st multitest
assert structure[0]["entries"][0]["name"] == "Alpha" # 1st suite name
assert len(structure[0]["entries"][0]["entries"]) == 3 # 3 testcases
assert structure[1]["name"] == "Secondary" # 2nd multitest name
assert len(structure[1]["entries"]) == 1 # one suite in 2nd multitest
assert structure[1]["entries"][0]["name"] == "Beta" # 1st suite name
assert len(structure[1]["entries"][0]["entries"]) == 2 # 2 testcases
assert (
len(structure[0]["entries"][0]["entries"]) == 4
) # 3 testcases, 1 synthesized
assert (
len(structure[0]["entries"][0]["entries"][2]["entries"]) == 3
) # 3 parametrized testcases

assert len(assertions) == 5 # 5 assertions in total
assert structure[1]["name"] == "Secondary" # 2nd multitest name
assert (
len(structure[1]["entries"]) == 2
) # one suite in 2nd multitest, 1 synthesized
assert structure[1]["entries"][0]["name"] == "After Start"
assert len(structure[1]["entries"][0]["entries"]) == 1
assert structure[1]["entries"][1]["name"] == "Beta" # 1st suite name
assert len(structure[1]["entries"][1]["entries"]) == 2 # 2 testcases

assert len(assertions) == 9 # 9 assertions in total
# only one assertion in each testcase in suite `Alpha`
assert assertions["test_comparison"][0]["type"] == "Equal"
assert assertions["test_membership"][0]["type"] == "Contain"
assert assertions["test_membership__arg_1"][0]["type"] == "Contain"
assert assertions["test_membership__arg_2"][0]["type"] == "Contain"
assert assertions["test_membership__arg_3"][0]["type"] == "Contain"
assert assertions["test_attach"][0]["type"] == "Attachment"
# 2 assertions in testcase `test_failure`
assert assertions["test_failure"][0]["type"] == "Equal"
assert assertions["test_failure"][1]["type"] == "NotEqual"
# no assertion in testcase `test_error`
assert len(assertions["test_error"]) == 0
# 2 assertions in synthesized cases, i.e. custom hooks
assert assertions["setup"][0]["type"] == "Log"
assert assertions["secondary_after_start"][0]["type"] == "Log"


def test_implicit_exporter_initialization(runpath):
Expand Down
Loading

0 comments on commit fe16fa4

Please sign in to comment.