Skip to content

Commit

Permalink
Add bounds and warning checks to test case generators (#99068)
Browse files Browse the repository at this point in the history 
This expands on Nik's work to centralize test case generation. In order to avoid needing to write extremely specific oracle functions, which can degenerate into re-implementations of the code under test, we adapt the data generation functions to accept a range of values, and optionally a collection of expected warnings. This will let test writers define tests in terms of ranges where we expect specific behavior (e.g. sqrt can define one test that expects null for all negative values, and another that expects square roots for only positive values).
  • Loading branch information
@not-napoleon
not-napoleon committed Aug 31, 2023
1 parent 3284903 commit b105b4a
Show file tree
Hide file tree
Showing 69 changed files with 1,329 additions and 788 deletions.
467 changes: 46 additions & 421 deletions .../test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
View file

Large diffs are not rendered by default.

590 changes: 590 additions & 0 deletions ...esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
View file

Large diffs are not rendered by default.

68 changes: 31 additions & 37 deletions ...t/java/org/elasticsearch/xpack/esql/expression/function/scalar/VaragsTestCaseBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

import org.apache.lucene.util.BytesRef;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.elasticsearch.xpack.ql.type.DataType;
import org.elasticsearch.xpack.ql.type.DataTypes;
import org.hamcrest.Matcher;
Expand Down Expand Up @@ -179,8 +179,8 @@ public VaragsTestCaseBuilder expectFlattenedBoolean(Function<Stream<Boolean>, Op
return this;
}

public List<AbstractFunctionTestCase.TestCaseSupplier> suppliers() {
List<AbstractFunctionTestCase.TestCaseSupplier> suppliers = new ArrayList<>();
public List<TestCaseSupplier> suppliers() {
List<TestCaseSupplier> suppliers = new ArrayList<>();
// TODO more types
if (expectedStr != null) {
strings(suppliers);
Expand All @@ -197,19 +197,19 @@ public List<AbstractFunctionTestCase.TestCaseSupplier> suppliers() {
return suppliers;
}

private void strings(List<AbstractFunctionTestCase.TestCaseSupplier> suppliers) {
private void strings(List<TestCaseSupplier> suppliers) {
for (int count = 1; count < MAX_WIDTH; count++) {
for (boolean multivalued : new boolean[] { false, true }) {
int paramCount = count;
suppliers.add(
new AbstractFunctionTestCase.TestCaseSupplier(
new TestCaseSupplier(
testCaseName(paramCount, multivalued, DataTypes.KEYWORD),
dataTypes(paramCount, DataTypes.KEYWORD),
() -> stringCase(DataTypes.KEYWORD, paramCount, multivalued)
)
);
suppliers.add(
new AbstractFunctionTestCase.TestCaseSupplier(
new TestCaseSupplier(
testCaseName(paramCount, multivalued, DataTypes.TEXT),
dataTypes(paramCount, DataTypes.TEXT),
() -> stringCase(DataTypes.TEXT, paramCount, multivalued)
Expand All @@ -219,29 +219,27 @@ private void strings(List<AbstractFunctionTestCase.TestCaseSupplier> suppliers)
}
}

private AbstractFunctionTestCase.TestCase stringCase(DataType dataType, int paramCount, boolean multivalued) {
private TestCaseSupplier.TestCase stringCase(DataType dataType, int paramCount, boolean multivalued) {
String[][] data = new String[paramCount][];
List<AbstractFunctionTestCase.TypedData> typedData = new ArrayList<>(paramCount);
List<TestCaseSupplier.TypedData> typedData = new ArrayList<>(paramCount);
for (int p = 0; p < paramCount; p++) {
if (multivalued) {
data[p] = ESTestCase.randomList(1, 4, () -> ESTestCase.randomAlphaOfLength(5)).toArray(String[]::new);
typedData.add(
new AbstractFunctionTestCase.TypedData(Arrays.stream(data[p]).map(BytesRef::new).toList(), dataType, "field" + p)
);
typedData.add(new TestCaseSupplier.TypedData(Arrays.stream(data[p]).map(BytesRef::new).toList(), dataType, "field" + p));
} else {
data[p] = new String[] { ESTestCase.randomAlphaOfLength(5) };
typedData.add(new AbstractFunctionTestCase.TypedData(new BytesRef(data[p][0]), dataType, "field" + p));
typedData.add(new TestCaseSupplier.TypedData(new BytesRef(data[p][0]), dataType, "field" + p));
}
}
return testCase(typedData, expectedEvaluatorPrefix.apply("BytesRef"), dataType, expectedStr.apply(data));
}

private void longs(List<AbstractFunctionTestCase.TestCaseSupplier> suppliers) {
private void longs(List<TestCaseSupplier> suppliers) {
for (int count = 1; count < MAX_WIDTH; count++) {
for (boolean multivalued : new boolean[] { false, true }) {
int paramCount = count;
suppliers.add(
new AbstractFunctionTestCase.TestCaseSupplier(
new TestCaseSupplier(
testCaseName(paramCount, multivalued, DataTypes.LONG),
dataTypes(paramCount, DataTypes.LONG),
() -> longCase(paramCount, multivalued)
Expand All @@ -251,34 +249,30 @@ private void longs(List<AbstractFunctionTestCase.TestCaseSupplier> suppliers) {
}
}

private AbstractFunctionTestCase.TestCase longCase(int paramCount, boolean multivalued) {
private TestCaseSupplier.TestCase longCase(int paramCount, boolean multivalued) {
long[][] data = new long[paramCount][];
List<AbstractFunctionTestCase.TypedData> typedData = new ArrayList<>(paramCount);
List<TestCaseSupplier.TypedData> typedData = new ArrayList<>(paramCount);
for (int p = 0; p < paramCount; p++) {
if (multivalued) {
List<Long> d = ESTestCase.randomList(1, 4, () -> ESTestCase.randomLong());
data[p] = d.stream().mapToLong(Long::longValue).toArray();
typedData.add(
new AbstractFunctionTestCase.TypedData(
Arrays.stream(data[p]).mapToObj(Long::valueOf).toList(),
DataTypes.LONG,
"field" + p
)
new TestCaseSupplier.TypedData(Arrays.stream(data[p]).mapToObj(Long::valueOf).toList(), DataTypes.LONG, "field" + p)
);
} else {
data[p] = new long[] { ESTestCase.randomLong() };
typedData.add(new AbstractFunctionTestCase.TypedData(data[p][0], DataTypes.LONG, "field" + p));
typedData.add(new TestCaseSupplier.TypedData(data[p][0], DataTypes.LONG, "field" + p));
}
}
return testCase(typedData, expectedEvaluatorPrefix.apply("Long"), DataTypes.LONG, expectedLong.apply(data));
}

private void ints(List<AbstractFunctionTestCase.TestCaseSupplier> suppliers) {
private void ints(List<TestCaseSupplier> suppliers) {
for (int count = 1; count < MAX_WIDTH; count++) {
for (boolean multivalued : new boolean[] { false, true }) {
int paramCount = count;
suppliers.add(
new AbstractFunctionTestCase.TestCaseSupplier(
new TestCaseSupplier(
testCaseName(paramCount, multivalued, DataTypes.INTEGER),
dataTypes(paramCount, DataTypes.INTEGER),
() -> intCase(paramCount, multivalued)
Expand All @@ -288,28 +282,28 @@ private void ints(List<AbstractFunctionTestCase.TestCaseSupplier> suppliers) {
}
}

private AbstractFunctionTestCase.TestCase intCase(int paramCount, boolean multivalued) {
private TestCaseSupplier.TestCase intCase(int paramCount, boolean multivalued) {
int[][] data = new int[paramCount][];
List<AbstractFunctionTestCase.TypedData> typedData = new ArrayList<>(paramCount);
List<TestCaseSupplier.TypedData> typedData = new ArrayList<>(paramCount);
for (int p = 0; p < paramCount; p++) {
if (multivalued) {
List<Integer> d = ESTestCase.randomList(1, 4, () -> ESTestCase.randomInt());
data[p] = d.stream().mapToInt(Integer::intValue).toArray();
typedData.add(new AbstractFunctionTestCase.TypedData(d, DataTypes.INTEGER, "field" + p));
typedData.add(new TestCaseSupplier.TypedData(d, DataTypes.INTEGER, "field" + p));
} else {
data[p] = new int[] { ESTestCase.randomInt() };
typedData.add(new AbstractFunctionTestCase.TypedData(data[p][0], DataTypes.INTEGER, "field" + p));
typedData.add(new TestCaseSupplier.TypedData(data[p][0], DataTypes.INTEGER, "field" + p));
}
}
return testCase(typedData, expectedEvaluatorPrefix.apply("Int"), DataTypes.INTEGER, expectedInt.apply(data));
}

private void booleans(List<AbstractFunctionTestCase.TestCaseSupplier> suppliers) {
private void booleans(List<TestCaseSupplier> suppliers) {
for (int count = 1; count < MAX_WIDTH; count++) {
for (boolean multivalued : new boolean[] { false, true }) {
int paramCount = count;
suppliers.add(
new AbstractFunctionTestCase.TestCaseSupplier(
new TestCaseSupplier(
testCaseName(paramCount, multivalued, DataTypes.BOOLEAN),
dataTypes(paramCount, DataTypes.BOOLEAN),
() -> booleanCase(paramCount, multivalued)
Expand All @@ -319,9 +313,9 @@ private void booleans(List<AbstractFunctionTestCase.TestCaseSupplier> suppliers)
}
}

private AbstractFunctionTestCase.TestCase booleanCase(int paramCount, boolean multivalued) {
private TestCaseSupplier.TestCase booleanCase(int paramCount, boolean multivalued) {
boolean[][] data = new boolean[paramCount][];
List<AbstractFunctionTestCase.TypedData> typedData = new ArrayList<>(paramCount);
List<TestCaseSupplier.TypedData> typedData = new ArrayList<>(paramCount);
for (int p = 0; p < paramCount; p++) {
if (multivalued) {
int size = ESTestCase.between(1, 5);
Expand All @@ -331,10 +325,10 @@ private AbstractFunctionTestCase.TestCase booleanCase(int paramCount, boolean mu
data[p][i] = ESTestCase.randomBoolean();
paramData.add(data[p][i]);
}
typedData.add(new AbstractFunctionTestCase.TypedData(paramData, DataTypes.BOOLEAN, "field" + p));
typedData.add(new TestCaseSupplier.TypedData(paramData, DataTypes.BOOLEAN, "field" + p));
} else {
data[p] = new boolean[] { ESTestCase.randomBoolean() };
typedData.add(new AbstractFunctionTestCase.TypedData(data[p][0], DataTypes.BOOLEAN, "field" + p));
typedData.add(new TestCaseSupplier.TypedData(data[p][0], DataTypes.BOOLEAN, "field" + p));
}
}
return testCase(typedData, expectedEvaluatorPrefix.apply("Boolean"), DataTypes.BOOLEAN, expectedBoolean.apply(data));
Expand All @@ -348,13 +342,13 @@ private String testCaseName(int count, boolean multivalued, DataType type) {
+ ")";
}

protected AbstractFunctionTestCase.TestCase testCase(
List<AbstractFunctionTestCase.TypedData> typedData,
protected TestCaseSupplier.TestCase testCase(
List<TestCaseSupplier.TypedData> typedData,
String expectedEvaluatorPrefix,
DataType expectedType,
Matcher<Object> expectedValue
) {
return new AbstractFunctionTestCase.TestCase(
return new TestCaseSupplier.TestCase(
typedData,
expectedToString(expectedEvaluatorPrefix, typedData.size()),
expectedType,
Expand Down
13 changes: 7 additions & 6 deletions ...t/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.elasticsearch.compute.data.IntBlock;
import org.elasticsearch.compute.data.Page;
import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Expression.TypeResolution;
Expand All @@ -33,7 +34,7 @@

public class CaseTests extends AbstractFunctionTestCase {

public CaseTests(@Name("TestCase") Supplier<TestCase> testCaseSupplier) {
public CaseTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
this.testCase = testCaseSupplier.get();
}

Expand All @@ -43,12 +44,12 @@ public CaseTests(@Name("TestCase") Supplier<TestCase> testCaseSupplier) {
@ParametersFactory
public static Iterable<Object[]> parameters() {
return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("basics", () -> {
List<TypedData> typedData = List.of(
new TypedData(true, DataTypes.BOOLEAN, "cond"),
new TypedData(new BytesRef("a"), DataTypes.KEYWORD, "a"),
new TypedData(new BytesRef("b"), DataTypes.KEYWORD, "b")
List<TestCaseSupplier.TypedData> typedData = List.of(
new TestCaseSupplier.TypedData(true, DataTypes.BOOLEAN, "cond"),
new TestCaseSupplier.TypedData(new BytesRef("a"), DataTypes.KEYWORD, "a"),
new TestCaseSupplier.TypedData(new BytesRef("b"), DataTypes.KEYWORD, "b")
);
return new TestCase(
return new TestCaseSupplier.TestCase(
typedData,
"CaseEvaluator[resultType=BYTES_REF, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
+ "value=Attribute[channel=1]]], elseVal=Attribute[channel=2]]",
Expand Down
9 changes: 5 additions & 4 deletions ...va/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

import org.apache.lucene.util.BytesRef;
import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.elasticsearch.xpack.esql.expression.function.scalar.VaragsTestCaseBuilder;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.tree.Source;
Expand All @@ -26,7 +27,7 @@
import static org.hamcrest.Matchers.equalTo;

public class GreatestTests extends AbstractFunctionTestCase {
public GreatestTests(@Name("TestCase") Supplier<TestCase> testCaseSupplier) {
public GreatestTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
this.testCase = testCaseSupplier.get();
}

Expand All @@ -43,10 +44,10 @@ public static Iterable<Object[]> parameters() {
new TestCaseSupplier(
"(a, b)",
List.of(DataTypes.KEYWORD, DataTypes.KEYWORD),
() -> new TestCase(
() -> new TestCaseSupplier.TestCase(
List.of(
new TypedData(new BytesRef("a"), DataTypes.KEYWORD, "a"),
new TypedData(new BytesRef("b"), DataTypes.KEYWORD, "b")
new TestCaseSupplier.TypedData(new BytesRef("a"), DataTypes.KEYWORD, "a"),
new TestCaseSupplier.TypedData(new BytesRef("b"), DataTypes.KEYWORD, "b")
),
"GreatestBytesRefEvaluator[values=[MvMax[field=Attribute[channel=0]], MvMax[field=Attribute[channel=1]]]]",
DataTypes.KEYWORD,
Expand Down
9 changes: 5 additions & 4 deletions .../java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

import org.apache.lucene.util.BytesRef;
import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.elasticsearch.xpack.esql.expression.function.scalar.VaragsTestCaseBuilder;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.tree.Source;
Expand All @@ -25,7 +26,7 @@
import static org.hamcrest.Matchers.equalTo;

public class LeastTests extends AbstractFunctionTestCase {
public LeastTests(@Name("TestCase") Supplier<TestCase> testCaseSupplier) {
public LeastTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
this.testCase = testCaseSupplier.get();
}

Expand All @@ -42,10 +43,10 @@ public static Iterable<Object[]> parameters() {
new TestCaseSupplier(
"(a, b)",
List.of(DataTypes.KEYWORD, DataTypes.KEYWORD),
() -> new TestCase(
() -> new TestCaseSupplier.TestCase(
List.of(
new TypedData(new BytesRef("a"), DataTypes.KEYWORD, "a"),
new TypedData(new BytesRef("b"), DataTypes.KEYWORD, "b")
new TestCaseSupplier.TypedData(new BytesRef("a"), DataTypes.KEYWORD, "a"),
new TestCaseSupplier.TypedData(new BytesRef("b"), DataTypes.KEYWORD, "b")
),
"LeastBytesRefEvaluator[values=[MvMin[field=Attribute[channel=0]], MvMin[field=Attribute[channel=1]]]]",
DataTypes.KEYWORD,
Expand Down
9 changes: 5 additions & 4 deletions ...t/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

import org.apache.lucene.util.BytesRef;
import org.elasticsearch.xpack.esql.EsqlTestUtils;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Literal;
Expand All @@ -29,17 +30,17 @@
import static org.hamcrest.Matchers.is;

public class DateExtractTests extends AbstractScalarFunctionTestCase {
public DateExtractTests(@Name("TestCase") Supplier<TestCase> testCaseSupplier) {
public DateExtractTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
this.testCase = testCaseSupplier.get();
}

@ParametersFactory
public static Iterable<Object[]> parameters() {
return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Date Extract Year", () -> {
return new TestCase(
return new TestCaseSupplier.TestCase(
List.of(
new TypedData(1687944333000L, DataTypes.DATETIME, "date"),
new TypedData(new BytesRef("YEAR"), DataTypes.KEYWORD, "field")
new TestCaseSupplier.TypedData(1687944333000L, DataTypes.DATETIME, "date"),
new TestCaseSupplier.TypedData(new BytesRef("YEAR"), DataTypes.KEYWORD, "field")
),
"DateExtractEvaluator[value=Attribute[channel=0], chronoField=Attribute[channel=1], zone=Z]",
DataTypes.LONG,
Expand Down
9 changes: 5 additions & 4 deletions ...est/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;

import org.apache.lucene.util.BytesRef;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.tree.Source;
Expand All @@ -23,17 +24,17 @@
import static org.hamcrest.Matchers.equalTo;

public class DateParseTests extends AbstractScalarFunctionTestCase {
public DateParseTests(@Name("TestCase") Supplier<TestCase> testCaseSupplier) {
public DateParseTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
this.testCase = testCaseSupplier.get();
}

@ParametersFactory
public static Iterable<Object[]> parameters() {
return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Basic Case", () -> {
return new TestCase(
return new TestCaseSupplier.TestCase(
List.of(
new TypedData(new BytesRef("2023-05-05"), DataTypes.KEYWORD, "first"),
new TypedData(new BytesRef("yyyy-MM-dd"), DataTypes.KEYWORD, "second")
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataTypes.KEYWORD, "first"),
new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataTypes.KEYWORD, "second")
),
"DateParseEvaluator[val=Attribute[channel=0], formatter=Attribute[channel=1], zoneId=Z]",
DataTypes.DATETIME,
Expand Down

0 comments on commit b105b4a

Please sign in to comment.