Skip to content
Permalink
Browse files
Added an option for float / double precision that would help to solve…
… precision related test failures (#594)
  • Loading branch information
agozhiy authored and Agirish committed Aug 13, 2019
1 parent c251ec3 commit 210a918f77785746a2ec3d6834076dd508cbd1bd
Showing 7 changed files with 26 additions and 10 deletions.
@@ -12,6 +12,7 @@
"schema": "mfs.tpch_sf1_maprdb_json",
"output-format": "tsv",
"expected-file": ".*.e_tsv",
"double-precision": 1.0E-11,
"verification-type": [
"in-memory"
]
@@ -12,6 +12,7 @@
"schema": "mfs.tpch_sf1_maprdb_json",
"output-format": "tsv",
"expected-file": ".*.e_tsv",
"double-precision": 1.0E-11,
"verification-type": [
"in-memory"
]
@@ -17,6 +17,8 @@
*/
package org.apache.drill.test.framework;

import org.apache.drill.test.framework.TestCaseModeler.TestMatrix;

import java.math.BigDecimal;
import java.sql.Types;
import java.util.List;
@@ -30,12 +32,18 @@
public class ColumnList {
private final List<Object> values;
private final List<Integer> types;
private final TestMatrix matrix;
private final boolean Simba;
public static final String SIMBA_JDBC = "sjdbc";

public ColumnList(List<Integer> types, List<Object> values) {
this(types, values, null);
}

public ColumnList(List<Integer> types, List<Object> values, TestMatrix matrix) {
this.values = values;
this.types = types;
this.matrix = matrix;
if (TestDriver.cmdParam.driverExt != null &&
TestDriver.cmdParam.driverExt.equals(ColumnList.SIMBA_JDBC)) {
this.Simba = true;
@@ -155,7 +163,7 @@ private boolean compare(ColumnList o1, ColumnList o2) {
float f1 = (Float) list1.get(i);
float f2 = (Float) list2.get(i);
if ((f1 + f2) / 2 != 0) {
if (!(Math.abs((f1 - f2) / ((f1 + f2) / 2)) < 1.0E-6)) return false;
return (Math.abs((f1 - f2) / ((f1 + f2) / 2)) < matrix.floatPrecision);
} else if (f1 != 0) {
return false;
}
@@ -168,7 +176,7 @@ private boolean compare(ColumnList o1, ColumnList o2) {
// otherwise proceed with "loosened" logic
if (!d1.equals(d2)) {
if ((d1 + d2) / 2 != 0) {
if (!(Math.abs((d1 - d2) / ((d1 + d2) / 2)) < 1.0E-12)) return false;
return (Math.abs((d1 - d2) / ((d1 + d2) / 2)) < matrix.doublePrecision);
} else if (d1 != 0) {
return false;
}
@@ -115,7 +115,7 @@ public void run() {
executeQuery(query);

if (getTestStatus()!= testStatus.CANCELED) { //Not to verify again if deliberately cancelled
testVerifier = new TestVerifier(columnTypes, query, columnLabels, matrix.verificationTypes);
testVerifier = new TestVerifier(columnTypes, query, columnLabels, matrix);
if (query.startsWith("explain") || matrix.verificationTypes.get(0).equalsIgnoreCase("regex") ||
matrix.verificationTypes.get(0).equalsIgnoreCase("regex-no-order") ||
matrix.verificationTypes.get(0).equalsIgnoreCase("filter-ratio")) {
@@ -253,7 +253,7 @@ private void executeQuery(String query) throws IOException, SQLException {
if (resultSet != null) {
while (resultSet.next()) {
List<Object> values = Utils.getRowValues(resultSet);
ColumnList columnList = new ColumnList(columnTypes, values);
ColumnList columnList = new ColumnList(columnTypes, values, matrix);
writer.write(columnList + "\n");
}
}
@@ -107,7 +107,7 @@ public void run() {

switch (cmdConsOut.exitCode) {
case 0:
TestVerifier testVerifier = new TestVerifier(columnTypes, query, columnLabels, matrix.verificationTypes);
TestVerifier testVerifier = new TestVerifier(columnTypes, query, columnLabels, matrix);
try {
if (query.startsWith("explain") || matrix.verificationTypes.get(0).equalsIgnoreCase("regex") ||
matrix.verificationTypes.get(0).equalsIgnoreCase("regex-no-order") ||
@@ -91,6 +91,10 @@ public static class TestMatrix {
// TODO: The below code can be reused when we decide to have username & password at suite level
public String username = DrillTestDefaults.USERNAME;
public String password = DrillTestDefaults.PASSWORD;
@JsonProperty("float-precision")
public double floatPrecision = 1.0E-6;
@JsonProperty("double-precision")
public double doublePrecision = 1.0E-12;
@JsonProperty("verification-type")
public List<String> verificationTypes;

@@ -41,6 +41,7 @@

import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.Lists;
import org.apache.drill.test.framework.TestCaseModeler.TestMatrix;
import org.apache.log4j.Logger;

/**
@@ -57,7 +58,7 @@ public class TestVerifier {
private List<Integer> types = null;
private String query;
private List<String> columnLabels;
private List<String> verificationTypes;
private TestMatrix matrix;
private boolean checkType = true;

public enum TestStatus {
@@ -67,11 +68,11 @@ public enum TestStatus {

public TestVerifier(List<Integer> types, String query,
List<String> columnLabels,
List<String> verificationType) {
TestMatrix matrix) {
this.types = types;
this.query = query;
this.columnLabels = columnLabels;
this.verificationTypes = verificationType;
this.matrix = matrix;
}

public TestVerifier() {
@@ -196,7 +197,7 @@ private Map<ColumnList, Integer> loadFromFileToMap(String filename)
* Detects the number of unexpected entries in the actual map
*
* @param count
* value of a particular entry in actual map
* value of a particular entry in actual map
* @return unexpected count for that entry
*/
private int getUnexpectedCount(Map<ColumnList, Integer> map, Map.Entry<ColumnList, Integer> entry){
@@ -280,7 +281,7 @@ private Map<ColumnList, Integer> loadFromFileToMap(String filename, boolean orde
typedFields.add(fields[i]);
}
}
ColumnList cl = new ColumnList(types, typedFields);
ColumnList cl = new ColumnList(types, typedFields, matrix);
if (ordered) {
resultSet.add(cl);
} else {
@@ -699,6 +700,7 @@ public TestStatus verifyTextPlan(String expectedOutput,

String actual = new String(Files.readAllBytes(Paths.get(actualOutput)));
boolean verified = false;
List<String> verificationTypes = matrix.verificationTypes;
if (verificationTypes.get(0).equalsIgnoreCase("regex")) {
verified = matchesAll(actual, expected);
} else if (verificationTypes.get(0).equalsIgnoreCase("regex-no-order")) {

0 comments on commit 210a918

Please sign in to comment.