Skip to content

matchers: containExactly displays potential suspects #1552

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 5, 2023
Merged

matchers: containExactly displays potential suspects #1552

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions ...test/groovy/org/testingisdocumenting/webtau/data/datanode/ValueExtractorByPathTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.testingisdocumenting.webtau.data.datanode

import org.junit.Test
import org.testingisdocumenting.webtau.data.ValuePath

class ValueExtractorByPathTest {
@Test
Expand Down Expand Up @@ -45,4 +46,12 @@ class ValueExtractorByPathTest {
ValueExtractorByPath.extractFromMapOrList([parent: [[child: 100], [another: 200]]], "parent[1].another.value")
.should == null
}

@Test
void "starts with"() {
new ValuePath("myActual[0].x").startsWith(new ValuePath("myActual[0]")).should == true
new ValuePath("myActual[0][1]").startsWith(new ValuePath("myActual[0]")).should == true
new ValuePath("myActual[0]x").startsWith(new ValuePath("myActual[0]")).should == false
new ValuePath("myActual[0]").startsWith(new ValuePath("myActual[0]")).should == false
}
}
4 changes: 4 additions & 0 deletions webtau-core/src/main/java/org/testingisdocumenting/webtau/data/ValuePath.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ public ValuePath(String path) {
this.path = path;
}

public boolean startsWith(ValuePath valuePath) {
return path.startsWith(valuePath.path + ".") || path.startsWith(valuePath.path + "[");
}

public ValuePath property(String propName) {
return new ValuePath(isEmpty() ? propName : path + "." + propName);
}
Expand Down
101 changes: 94 additions & 7 deletions .../main/java/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,27 @@
import org.testingisdocumenting.webtau.expectation.ExpectedValuesAware;
import org.testingisdocumenting.webtau.expectation.ValueMatcher;
import org.testingisdocumenting.webtau.expectation.equality.CompareToComparator;
import org.testingisdocumenting.webtau.expectation.equality.CompareToResult;
import org.testingisdocumenting.webtau.expectation.equality.ValuePathMessage;
import org.testingisdocumenting.webtau.reporter.TokenizedMessage;

import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.testingisdocumenting.webtau.WebTauCore.*;
import static org.testingisdocumenting.webtau.expectation.TokenizedReportUtils.*;

public class ContainExactlyMatcher implements ValueMatcher, ExpectedValuesAware, PrettyPrintable {
private final Collection<Object> expectedList;
private List<ValuePathWithValue<Object>> actualCopy;
private List<ValuePathWithValue<Object>> expectedCopy;

private final Map<ValuePath, List<List<ValuePathMessage>>> notEqualMessagesByExpectedPath = new HashMap<>();
private final List<ValuePathMessage> notEqualCandidateMessages = new ArrayList<>();
private final List<ValuePathMessage> missingMessages = new ArrayList<>();
private final List<ValuePathMessage> extraMessages = new ArrayList<>();

private CompareToComparator comparator;

public ContainExactlyMatcher(Collection<Object> expected) {
Expand All @@ -55,7 +63,13 @@ public Stream<Object> expectedValues() {

@Override
public Set<ValuePath> mismatchedPaths() {
return actualCopy.stream().map(ValuePathWithValue::getPath).collect(Collectors.toSet());
Set<ValuePath> potentialPaths = Stream.concat(missingMessages.stream().map(ValuePathMessage::getActualPath),
Stream.concat(extraMessages.stream().map(ValuePathMessage::getActualPath),
notEqualCandidateMessages.stream().map(ValuePathMessage::getActualPath)))
.collect(Collectors.toSet());
return potentialPaths.isEmpty() ?
actualCopy.stream().map(ValuePathWithValue::getPath).collect(Collectors.toSet()) :
potentialPaths;
}

@Override
Expand All @@ -77,20 +91,26 @@ public TokenizedMessage mismatchedTokenizedMessage(ValuePath actualPath, Object
expectedCopy.stream().map(ValuePathWithValue::getValue).toList());
}

if (!actualCopy.isEmpty()) {
if (!actualCopy.isEmpty() && notEqualCandidateMessages.isEmpty()) {
if (!messageTokens.isEmpty()) {
messageTokens = messageTokens.newLine();
}
messageTokens = messageTokens.error("unexpected elements").colon().value(
actualCopy.stream().map(ValuePathWithValue::getValue).toList());
}

if (!notEqualCandidateMessages.isEmpty() || !missingMessages.isEmpty() || !extraMessages.isEmpty()) {
messageTokens = messageTokens.newLine().add(generatePossibleMismatchesReport(actualPath));
}

return messageTokens;
}

@Override
public boolean matches(ValuePath actualPath, Object actualIterable) {
return matches(comparator, actualPath, actualIterable);
boolean result = matches(comparator, actualPath, actualIterable, true);
notEqualCandidateMessages.addAll(extractPotentialNotEqualMessages());
return result;
}

@Override
Expand All @@ -111,7 +131,7 @@ public TokenizedMessage negativeMismatchedTokenizedMessage(ValuePath actualPath,

@Override
public boolean negativeMatches(ValuePath actualPath, Object actualIterable) {
return !matches(comparator, actualPath, actualIterable);
return !matches(comparator, actualPath, actualIterable, false);
}

@Override
Expand All @@ -128,7 +148,7 @@ public String toString() {
}

@SuppressWarnings("unchecked")
private boolean matches(CompareToComparator comparator, ValuePath actualPath, Object actualIterable) {
private boolean matches(CompareToComparator comparator, ValuePath actualPath, Object actualIterable, boolean collectSuspects) {
if (!(actualIterable instanceof Iterable)) {
return false;
}
Expand All @@ -140,17 +160,84 @@ private boolean matches(CompareToComparator comparator, ValuePath actualPath, Ob
while (expectedIt.hasNext()) {
ValuePathWithValue<Object> expected = expectedIt.next();
Iterator<ValuePathWithValue<Object>> actualIt = actualCopy.iterator();

// collect mismatches for each remaining actual value
// find elements with the largest number of mismatches
// remember those elements as suspects per expected value
List<CompareToResult> compareToResults = new ArrayList<>();
boolean found = false;
while (actualIt.hasNext()) {
ValuePathWithValue<Object> actual = actualIt.next();
boolean isEqual = comparator.compareIsEqual(actual.getPath(), actual.getValue(), expected.getValue());
if (isEqual) {
CompareToResult compareToResult = comparator.compareUsingEqualOnly(actual.getPath(), actual.getValue(), expected.getValue());
if (compareToResult.isEqual()) {
actualIt.remove();
expectedIt.remove();
found = true;
break;
}

compareToResults.add(compareToResult);
}

if (!found && collectSuspects) {
notEqualMessagesByExpectedPath.put(expected.getPath(),
compareToResults.stream().map(CompareToResult::getNotEqualMessages).toList());

compareToResults.forEach(r -> missingMessages.addAll(r.getMissingMessages()));
compareToResults.forEach(r -> extraMessages.addAll(r.getExtraMessages()));
}
}

return actualCopy.isEmpty() && expectedCopy.isEmpty();
}

private List<ValuePathMessage> extractPotentialNotEqualMessages() {
List<ValuePath> actualPaths = actualCopy.stream().map(ValuePathWithValue::getPath).toList();
List<ValuePathMessage> notEqualCandidateMessages = new ArrayList<>();
for (ValuePathWithValue<Object> expectedWithPath : expectedCopy) {
List<List<ValuePathMessage>> notEqualMessageBatches = notEqualMessagesByExpectedPath.get(expectedWithPath.getPath());
if (notEqualMessageBatches == null) {
continue;
}

// remove all the messages that were matched against eventually matched actual values
notEqualMessageBatches = notEqualMessageBatches.stream()
.filter(batch -> {
if (batch.isEmpty()) {
return false;
}

ValuePathMessage firstMessage = batch.get(0);
return actualPaths.stream().anyMatch(path -> firstMessage.getActualPath().startsWith(path));
})
.toList();


// need to find a subset that has the least amount of mismatches
// it will be a potential mismatch detail to display,
//
int minNumberOMismatches = notEqualMessageBatches.stream()
.map(List::size)
.min(Integer::compareTo).orElse(0);

List<List<ValuePathMessage>> messagesWithMinFailures = notEqualMessageBatches.stream()
.filter(v -> v.size() == minNumberOMismatches).toList();

if (notEqualMessageBatches.size() != messagesWithMinFailures.size()) {
messagesWithMinFailures.forEach(notEqualCandidateMessages::addAll);
}
}

return notEqualCandidateMessages;
}

private TokenizedMessage generatePossibleMismatchesReport(ValuePath topLevelActualPath) {
return combineReportParts(
generateReportPart(topLevelActualPath, tokenizedMessage().error("possible mismatches"),
Collections.singletonList(notEqualCandidateMessages)),
generateReportPart(topLevelActualPath, tokenizedMessage().error("missing values"),
Collections.singletonList(missingMessages)),
generateReportPart(topLevelActualPath, tokenizedMessage().error("extra values"),
Collections.singletonList(extraMessages)));
}
}
7 changes: 7 additions & 0 deletions ...e/src/main/java/org/testingisdocumenting/webtau/expectation/equality/CompareToResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
/*
* Copyright 2023 webtau maintainers
* Copyright 2019 TWO SIGMA OPEN SOURCE, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
Expand Down Expand Up @@ -27,6 +28,12 @@ public class CompareToResult {
private List<ValuePathMessage> missingMessages = new ArrayList<>();
private List<ValuePathMessage> extraMessages = new ArrayList<>();

public int numberOfMismatches() {
return notEqualMessages.size() +
greaterMessages.size() + lessMessages.size() +
missingMessages.size() + extraMessages.size();
}

public boolean isEqual() {
return notEqualMessages.isEmpty() && hasNoExtraAndNoMissing();
}
Expand Down
23 changes: 13 additions & 10 deletions ...rg/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherGroovyTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,18 @@ import static org.testingisdocumenting.webtau.WebTauCore.*
class ContainExactlyMatcherGroovyTest {
@Test
void matchRecordsAndMaps() {
// records-and-maps-example
def list = [new Person("id1", 3, 10),
new Person("id2", 4, 20),
new Person("id2", 4, 20)]

actual(list).should(containExactly(
[id: "id2", level: 4, monthsAtCompany: 20],
[id: "id1", level: 3, monthsAtCompany: 10],
[id: "id2", level: 4, monthsAtCompany: 20]))
// records-and-maps-example
code {
// possible-mismatches-example
def list = [
new Person("id1", 3, 12),
new Person("id1", 4, 10),
new Person("id2", 4, 20)
]
actual(list).should(containExactly(
[id: "id2", level: 4, monthsAtCompany: 20],
[id: "id1", level: 8, monthsAtCompany: 10],
[id: "id1", level: 7, monthsAtCompany: 12]))
// possible-mismatches-example
} should throwException(AssertionError)
}
}
98 changes: 98 additions & 0 deletions ...va/org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherJavaTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,104 @@ public void missingDuplicatedValueRecordsAndMaps() {
});
}

@Test
public void suspectCandidateValueRecordsAndMaps() {
runExpectExceptionCaptureAndValidateOutput(AssertionError.class, "possible-mismatches-output", """
X failed expecting [value] to contain exactly [
{"id": "id2", "level": 4, "monthsAtCompany": 20},
{"id": "id1", "level": 8, "monthsAtCompany": 10},
{"id": "id1", "level": 7, "monthsAtCompany": 12}
]:
no matches found for: [{"id": "id1", "level": 8, "monthsAtCompany": 10}, {"id": "id1", "level": 7, "monthsAtCompany": 12}]
possible mismatches:
\s
[value][1].level: actual: 4 <java.lang.Integer>
expected: 8 <java.lang.Integer>
[value][0].level: actual: 3 <java.lang.Integer>
expected: 7 <java.lang.Integer> (Xms)
\s
[
{"id": "id1", "level": **3**, "monthsAtCompany": 12},
{"id": "id1", "level": **4**, "monthsAtCompany": 10},
{"id": "id2", "level": 4, "monthsAtCompany": 20}
]""", () -> {

// possible-mismatches-example
List<?> list = list(
new Person("id1", 3, 12),
new Person("id1", 4, 10),
new Person("id2", 4, 20));

actual(list).should(containExactly(
map("id", "id2", "level", 4, "monthsAtCompany", 20),
map("id", "id1", "level", 8, "monthsAtCompany", 10),
map("id", "id1", "level", 7, "monthsAtCompany", 12)));
// possible-mismatches-example
});
}

@Test
public void suspectCandidateWithMissing() {
runExpectExceptionAndValidateOutput(AssertionError.class, """
X failed expecting [value] to contain exactly [
{"id": "id1", "level": 8, "monthsAtCompany": 10},
{"id": "id1", "level": 7, "monthsAtCompany": 12},
{"id": "id2", "level": 4, "monthsAtCompany": 20}
]:
no matches found for: [{"id": "id1", "level": 8, "monthsAtCompany": 10}]
unexpected elements: [{"id": "id1", "level": 5}]
missing values:
\s
[value][0].monthsAtCompany: 10 (Xms)
\s
[
{"id": "id1", "level": 5, "monthsAtCompany": **<missing>**},
{"id": "id1", "level": 7, "monthsAtCompany": 12},
{"id": "id2", "level": 4, "monthsAtCompany": 20}
]""", () -> {
List<?> list = list(
map("id", "id1", "level", 5),
map("id", "id1", "level", 7, "monthsAtCompany", 12),
map("id", "id2", "level", 4, "monthsAtCompany", 20));

actual(list).should(containExactly(
map("id", "id1", "level", 8, "monthsAtCompany", 10),
map("id", "id1", "level", 7, "monthsAtCompany", 12),
map("id", "id2", "level", 4, "monthsAtCompany", 20)));
});
}

@Test
public void suspectCandidateWithExtra() {
runExpectExceptionAndValidateOutput(AssertionError.class, """
X failed expecting [value] to contain exactly [
{"id": "id1", "level": 8, "monthsAtCompany": 10},
{"id": "id1", "level": 7, "monthsAtCompany": 12},
{"id": "id2", "level": 4, "monthsAtCompany": 20}
]:
no matches found for: [{"id": "id1", "level": 8, "monthsAtCompany": 10}]
unexpected elements: [{"id": "id1", "level": 5, "monthsAtCompany": 14, "salary": "yes"}]
extra values:
\s
[value][0].salary: "yes" (Xms)
\s
[
{"id": "id1", "level": 5, "monthsAtCompany": 14, "salary": **"yes"**},
{"id": "id1", "level": 7, "monthsAtCompany": 12},
{"id": "id2", "level": 4, "monthsAtCompany": 20}
]""", () -> {
List<?> list = list(
map("id", "id1", "level", 5, "monthsAtCompany", 14, "salary", "yes"),
map("id", "id1", "level", 7, "monthsAtCompany", 12),
map("id", "id2", "level", 4, "monthsAtCompany", 20));

actual(list).should(containExactly(
map("id", "id1", "level", 8, "monthsAtCompany", 10),
map("id", "id1", "level", 7, "monthsAtCompany", 12),
map("id", "id2", "level", 4, "monthsAtCompany", 20)));
});
}

@Test
public void mismatchValue() {
runExpectExceptionAndValidateOutput(AssertionError.class, """
Expand Down
11 changes: 9 additions & 2 deletions webtau-docs/znai/matchers/contain-exactly.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,19 @@ Use `:identifier: containExactly` to match two collections of elements in any or
```tabs
Groovy:
:include-file: org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherGroovyTest.groovy {
surroundedBy: "records-and-maps-example"
surroundedBy: "possible-mismatches-example"
}
Java:
:include-file: org/testingisdocumenting/webtau/expectation/contain/ContainExactlyMatcherJavaTest.java {
surroundedBy: "records-and-maps-example"
surroundedBy: "possible-mismatches-example"
}
```

Console output displays potential mismatches to help with investigation:

:include-cli-output: doc-artifacts/possible-mismatches-output.txt {
title: "console output"
}


Note: If you have a clear key column(s) defined, consider using `TableData` as [expected values](matchers/java-beans-and-records#java-beans-equal-table-data)