diff --git a/webtau-core-groovy/src/test/groovy/org/testingisdocumenting/webtau/data/datanode/ValueExtractorByPathTest.groovy b/webtau-core-groovy/src/test/groovy/org/testingisdocumenting/webtau/data/datanode/ValueExtractorByPathTest.groovy index 72d6e8156..c9e10b0f3 100644 --- a/webtau-core-groovy/src/test/groovy/org/testingisdocumenting/webtau/data/datanode/ValueExtractorByPathTest.groovy +++ b/webtau-core-groovy/src/test/groovy/org/testingisdocumenting/webtau/data/datanode/ValueExtractorByPathTest.groovy @@ -17,6 +17,7 @@ package org.testingisdocumenting.webtau.data.datanode import org.junit.Test +import org.testingisdocumenting.webtau.data.ValuePath class ValueExtractorByPathTest { @Test @@ -45,4 +46,12 @@ class ValueExtractorByPathTest { ValueExtractorByPath.extractFromMapOrList([parent: [[child: 100], [another: 200]]], "parent[1].another.value") .should == null } + + @Test + void "starts with"() { + new ValuePath("myActual[0].x").startsWith(new ValuePath("myActual[0]")).should == true + new ValuePath("myActual[0][1]").startsWith(new ValuePath("myActual[0]")).should == true + new ValuePath("myActual[0]x").startsWith(new ValuePath("myActual[0]")).should == false + new ValuePath("myActual[0]").startsWith(new ValuePath("myActual[0]")).should == false + } } diff --git a/webtau-core/src/main/java/org/testingisdocumenting/webtau/data/ValuePath.java b/webtau-core/src/main/java/org/testingisdocumenting/webtau/data/ValuePath.java index 5b33f059d..7790e10c5 100644 --- a/webtau-core/src/main/java/org/testingisdocumenting/webtau/data/ValuePath.java +++ b/webtau-core/src/main/java/org/testingisdocumenting/webtau/data/ValuePath.java @@ -32,6 +32,10 @@ public ValuePath(String path) { this.path = path; } + public boolean startsWith(ValuePath valuePath) { + return path.startsWith(valuePath.path + ".") || path.startsWith(valuePath.path + "["); + } + public ValuePath property(String propName) { return new ValuePath(isEmpty() ? propName : path + "." + propName); } diff --git a/webtau-core/src/main/java/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcher.java b/webtau-core/src/main/java/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcher.java index 0379db1b6..2bb8bb6f6 100644 --- a/webtau-core/src/main/java/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcher.java +++ b/webtau-core/src/main/java/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcher.java @@ -24,6 +24,8 @@ import org.testingisdocumenting.webtau.expectation.ExpectedValuesAware; import org.testingisdocumenting.webtau.expectation.ValueMatcher; import org.testingisdocumenting.webtau.expectation.equality.CompareToComparator; +import org.testingisdocumenting.webtau.expectation.equality.CompareToResult; +import org.testingisdocumenting.webtau.expectation.equality.ValuePathMessage; import org.testingisdocumenting.webtau.reporter.TokenizedMessage; import java.util.*; @@ -31,12 +33,18 @@ import java.util.stream.Stream; import static org.testingisdocumenting.webtau.WebTauCore.*; +import static org.testingisdocumenting.webtau.expectation.TokenizedReportUtils.*; public class ContainExactlyMatcher implements ValueMatcher, ExpectedValuesAware, PrettyPrintable { private final Collection expectedList; private List> actualCopy; private List> expectedCopy; + private final Map>> notEqualMessagesByExpectedPath = new HashMap<>(); + private final List notEqualCandidateMessages = new ArrayList<>(); + private final List missingMessages = new ArrayList<>(); + private final List extraMessages = new ArrayList<>(); + private CompareToComparator comparator; public ContainExactlyMatcher(Collection expected) { @@ -55,7 +63,13 @@ public Stream expectedValues() { @Override public Set mismatchedPaths() { - return actualCopy.stream().map(ValuePathWithValue::getPath).collect(Collectors.toSet()); + Set potentialPaths = Stream.concat(missingMessages.stream().map(ValuePathMessage::getActualPath), + Stream.concat(extraMessages.stream().map(ValuePathMessage::getActualPath), + notEqualCandidateMessages.stream().map(ValuePathMessage::getActualPath))) + .collect(Collectors.toSet()); + return potentialPaths.isEmpty() ? + actualCopy.stream().map(ValuePathWithValue::getPath).collect(Collectors.toSet()) : + potentialPaths; } @Override @@ -77,7 +91,7 @@ public TokenizedMessage mismatchedTokenizedMessage(ValuePath actualPath, Object expectedCopy.stream().map(ValuePathWithValue::getValue).toList()); } - if (!actualCopy.isEmpty()) { + if (!actualCopy.isEmpty() && notEqualCandidateMessages.isEmpty()) { if (!messageTokens.isEmpty()) { messageTokens = messageTokens.newLine(); } @@ -85,12 +99,18 @@ public TokenizedMessage mismatchedTokenizedMessage(ValuePath actualPath, Object actualCopy.stream().map(ValuePathWithValue::getValue).toList()); } + if (!notEqualCandidateMessages.isEmpty() || !missingMessages.isEmpty() || !extraMessages.isEmpty()) { + messageTokens = messageTokens.newLine().add(generatePossibleMismatchesReport(actualPath)); + } + return messageTokens; } @Override public boolean matches(ValuePath actualPath, Object actualIterable) { - return matches(comparator, actualPath, actualIterable); + boolean result = matches(comparator, actualPath, actualIterable, true); + notEqualCandidateMessages.addAll(extractPotentialNotEqualMessages()); + return result; } @Override @@ -111,7 +131,7 @@ public TokenizedMessage negativeMismatchedTokenizedMessage(ValuePath actualPath, @Override public boolean negativeMatches(ValuePath actualPath, Object actualIterable) { - return !matches(comparator, actualPath, actualIterable); + return !matches(comparator, actualPath, actualIterable, false); } @Override @@ -128,7 +148,7 @@ public String toString() { } @SuppressWarnings("unchecked") - private boolean matches(CompareToComparator comparator, ValuePath actualPath, Object actualIterable) { + private boolean matches(CompareToComparator comparator, ValuePath actualPath, Object actualIterable, boolean collectSuspects) { if (!(actualIterable instanceof Iterable)) { return false; } @@ -140,17 +160,84 @@ private boolean matches(CompareToComparator comparator, ValuePath actualPath, Ob while (expectedIt.hasNext()) { ValuePathWithValue expected = expectedIt.next(); Iterator> actualIt = actualCopy.iterator(); + + // collect mismatches for each remaining actual value + // find elements with the largest number of mismatches + // remember those elements as suspects per expected value + List compareToResults = new ArrayList<>(); + boolean found = false; while (actualIt.hasNext()) { ValuePathWithValue actual = actualIt.next(); - boolean isEqual = comparator.compareIsEqual(actual.getPath(), actual.getValue(), expected.getValue()); - if (isEqual) { + CompareToResult compareToResult = comparator.compareUsingEqualOnly(actual.getPath(), actual.getValue(), expected.getValue()); + if (compareToResult.isEqual()) { actualIt.remove(); expectedIt.remove(); + found = true; break; } + + compareToResults.add(compareToResult); + } + + if (!found && collectSuspects) { + notEqualMessagesByExpectedPath.put(expected.getPath(), + compareToResults.stream().map(CompareToResult::getNotEqualMessages).toList()); + + compareToResults.forEach(r -> missingMessages.addAll(r.getMissingMessages())); + compareToResults.forEach(r -> extraMessages.addAll(r.getExtraMessages())); } } return actualCopy.isEmpty() && expectedCopy.isEmpty(); } + + private List extractPotentialNotEqualMessages() { + List actualPaths = actualCopy.stream().map(ValuePathWithValue::getPath).toList(); + List notEqualCandidateMessages = new ArrayList<>(); + for (ValuePathWithValue expectedWithPath : expectedCopy) { + List> notEqualMessageBatches = notEqualMessagesByExpectedPath.get(expectedWithPath.getPath()); + if (notEqualMessageBatches == null) { + continue; + } + + // remove all the messages that were matched against eventually matched actual values + notEqualMessageBatches = notEqualMessageBatches.stream() + .filter(batch -> { + if (batch.isEmpty()) { + return false; + } + + ValuePathMessage firstMessage = batch.get(0); + return actualPaths.stream().anyMatch(path -> firstMessage.getActualPath().startsWith(path)); + }) + .toList(); + + + // need to find a subset that has the least amount of mismatches + // it will be a potential mismatch detail to display, + // + int minNumberOMismatches = notEqualMessageBatches.stream() + .map(List::size) + .min(Integer::compareTo).orElse(0); + + List> messagesWithMinFailures = notEqualMessageBatches.stream() + .filter(v -> v.size() == minNumberOMismatches).toList(); + + if (notEqualMessageBatches.size() != messagesWithMinFailures.size()) { + messagesWithMinFailures.forEach(notEqualCandidateMessages::addAll); + } + } + + return notEqualCandidateMessages; + } + + private TokenizedMessage generatePossibleMismatchesReport(ValuePath topLevelActualPath) { + return combineReportParts( + generateReportPart(topLevelActualPath, tokenizedMessage().error("possible mismatches"), + Collections.singletonList(notEqualCandidateMessages)), + generateReportPart(topLevelActualPath, tokenizedMessage().error("missing values"), + Collections.singletonList(missingMessages)), + generateReportPart(topLevelActualPath, tokenizedMessage().error("extra values"), + Collections.singletonList(extraMessages))); + } } diff --git a/webtau-core/src/main/java/org/testingisdocumenting/webtau/expectation/equality/CompareToResult.java b/webtau-core/src/main/java/org/testingisdocumenting/webtau/expectation/equality/CompareToResult.java index 0173a43a0..a8f208f6a 100644 --- a/webtau-core/src/main/java/org/testingisdocumenting/webtau/expectation/equality/CompareToResult.java +++ b/webtau-core/src/main/java/org/testingisdocumenting/webtau/expectation/equality/CompareToResult.java @@ -1,4 +1,5 @@ /* + * Copyright 2023 webtau maintainers * Copyright 2019 TWO SIGMA OPEN SOURCE, LLC * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,6 +28,12 @@ public class CompareToResult { private List missingMessages = new ArrayList<>(); private List extraMessages = new ArrayList<>(); + public int numberOfMismatches() { + return notEqualMessages.size() + + greaterMessages.size() + lessMessages.size() + + missingMessages.size() + extraMessages.size(); + } + public boolean isEqual() { return notEqualMessages.isEmpty() && hasNoExtraAndNoMissing(); } diff --git a/webtau-core/src/test/groovy/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherGroovyTest.groovy b/webtau-core/src/test/groovy/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherGroovyTest.groovy index 43b540d55..80704188a 100644 --- a/webtau-core/src/test/groovy/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherGroovyTest.groovy +++ b/webtau-core/src/test/groovy/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherGroovyTest.groovy @@ -24,15 +24,18 @@ import static org.testingisdocumenting.webtau.WebTauCore.* class ContainExactlyMatcherGroovyTest { @Test void matchRecordsAndMaps() { - // records-and-maps-example - def list = [new Person("id1", 3, 10), - new Person("id2", 4, 20), - new Person("id2", 4, 20)] - - actual(list).should(containExactly( - [id: "id2", level: 4, monthsAtCompany: 20], - [id: "id1", level: 3, monthsAtCompany: 10], - [id: "id2", level: 4, monthsAtCompany: 20])) - // records-and-maps-example + code { + // possible-mismatches-example + def list = [ + new Person("id1", 3, 12), + new Person("id1", 4, 10), + new Person("id2", 4, 20) + ] + actual(list).should(containExactly( + [id: "id2", level: 4, monthsAtCompany: 20], + [id: "id1", level: 8, monthsAtCompany: 10], + [id: "id1", level: 7, monthsAtCompany: 12])) + // possible-mismatches-example + } should throwException(AssertionError) } } diff --git a/webtau-core/src/test/java/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherJavaTest.java b/webtau-core/src/test/java/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherJavaTest.java index 3cfe85160..81bf965b1 100644 --- a/webtau-core/src/test/java/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherJavaTest.java +++ b/webtau-core/src/test/java/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherJavaTest.java @@ -79,6 +79,104 @@ public void missingDuplicatedValueRecordsAndMaps() { }); } + @Test + public void suspectCandidateValueRecordsAndMaps() { + runExpectExceptionCaptureAndValidateOutput(AssertionError.class, "possible-mismatches-output", """ + X failed expecting [value] to contain exactly [ + {"id": "id2", "level": 4, "monthsAtCompany": 20}, + {"id": "id1", "level": 8, "monthsAtCompany": 10}, + {"id": "id1", "level": 7, "monthsAtCompany": 12} + ]: + no matches found for: [{"id": "id1", "level": 8, "monthsAtCompany": 10}, {"id": "id1", "level": 7, "monthsAtCompany": 12}] + possible mismatches: + \s + [value][1].level: actual: 4 + expected: 8 + [value][0].level: actual: 3 + expected: 7 (Xms) + \s + [ + {"id": "id1", "level": **3**, "monthsAtCompany": 12}, + {"id": "id1", "level": **4**, "monthsAtCompany": 10}, + {"id": "id2", "level": 4, "monthsAtCompany": 20} + ]""", () -> { + + // possible-mismatches-example + List list = list( + new Person("id1", 3, 12), + new Person("id1", 4, 10), + new Person("id2", 4, 20)); + + actual(list).should(containExactly( + map("id", "id2", "level", 4, "monthsAtCompany", 20), + map("id", "id1", "level", 8, "monthsAtCompany", 10), + map("id", "id1", "level", 7, "monthsAtCompany", 12))); + // possible-mismatches-example + }); + } + + @Test + public void suspectCandidateWithMissing() { + runExpectExceptionAndValidateOutput(AssertionError.class, """ + X failed expecting [value] to contain exactly [ + {"id": "id1", "level": 8, "monthsAtCompany": 10}, + {"id": "id1", "level": 7, "monthsAtCompany": 12}, + {"id": "id2", "level": 4, "monthsAtCompany": 20} + ]: + no matches found for: [{"id": "id1", "level": 8, "monthsAtCompany": 10}] + unexpected elements: [{"id": "id1", "level": 5}] + missing values: + \s + [value][0].monthsAtCompany: 10 (Xms) + \s + [ + {"id": "id1", "level": 5, "monthsAtCompany": ****}, + {"id": "id1", "level": 7, "monthsAtCompany": 12}, + {"id": "id2", "level": 4, "monthsAtCompany": 20} + ]""", () -> { + List list = list( + map("id", "id1", "level", 5), + map("id", "id1", "level", 7, "monthsAtCompany", 12), + map("id", "id2", "level", 4, "monthsAtCompany", 20)); + + actual(list).should(containExactly( + map("id", "id1", "level", 8, "monthsAtCompany", 10), + map("id", "id1", "level", 7, "monthsAtCompany", 12), + map("id", "id2", "level", 4, "monthsAtCompany", 20))); + }); + } + + @Test + public void suspectCandidateWithExtra() { + runExpectExceptionAndValidateOutput(AssertionError.class, """ + X failed expecting [value] to contain exactly [ + {"id": "id1", "level": 8, "monthsAtCompany": 10}, + {"id": "id1", "level": 7, "monthsAtCompany": 12}, + {"id": "id2", "level": 4, "monthsAtCompany": 20} + ]: + no matches found for: [{"id": "id1", "level": 8, "monthsAtCompany": 10}] + unexpected elements: [{"id": "id1", "level": 5, "monthsAtCompany": 14, "salary": "yes"}] + extra values: + \s + [value][0].salary: "yes" (Xms) + \s + [ + {"id": "id1", "level": 5, "monthsAtCompany": 14, "salary": **"yes"**}, + {"id": "id1", "level": 7, "monthsAtCompany": 12}, + {"id": "id2", "level": 4, "monthsAtCompany": 20} + ]""", () -> { + List list = list( + map("id", "id1", "level", 5, "monthsAtCompany", 14, "salary", "yes"), + map("id", "id1", "level", 7, "monthsAtCompany", 12), + map("id", "id2", "level", 4, "monthsAtCompany", 20)); + + actual(list).should(containExactly( + map("id", "id1", "level", 8, "monthsAtCompany", 10), + map("id", "id1", "level", 7, "monthsAtCompany", 12), + map("id", "id2", "level", 4, "monthsAtCompany", 20))); + }); + } + @Test public void mismatchValue() { runExpectExceptionAndValidateOutput(AssertionError.class, """ diff --git a/webtau-docs/znai/matchers/contain-exactly.md b/webtau-docs/znai/matchers/contain-exactly.md index 9c81e6343..a36b87499 100644 --- a/webtau-docs/znai/matchers/contain-exactly.md +++ b/webtau-docs/znai/matchers/contain-exactly.md @@ -7,12 +7,19 @@ Use `:identifier: containExactly` to match two collections of elements in any or ```tabs Groovy: :include-file: org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherGroovyTest.groovy { - surroundedBy: "records-and-maps-example" + surroundedBy: "possible-mismatches-example" } Java: :include-file: org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherJavaTest.java { - surroundedBy: "records-and-maps-example" + surroundedBy: "possible-mismatches-example" } ``` +Console output displays potential mismatches to help with investigation: + +:include-cli-output: doc-artifacts/possible-mismatches-output.txt { + title: "console output" +} + + Note: If you have a clear key column(s) defined, consider using `TableData` as [expected values](matchers/java-beans-and-records#java-beans-equal-table-data) \ No newline at end of file