Fix #2873: DataQuality test specifications and APIs

open-metadata · Feb 24, 2022 · 5a63cc9 · 5a63cc9
1 parent ffee4b1
commit 5a63cc9
Show file tree

Hide file tree

Showing 19 changed files with 448 additions and 374 deletions.
diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java
@@ -46,10 +46,10 @@
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.catalog.entity.data.Database;
 import org.openmetadata.catalog.entity.data.Table;
-import org.openmetadata.catalog.entity.data.TableTest;
 import org.openmetadata.catalog.entity.services.DatabaseService;
 import org.openmetadata.catalog.exception.CatalogExceptionMessage;
 import org.openmetadata.catalog.resources.databases.TableResource;
+import org.openmetadata.catalog.tests.TableTest;
 import org.openmetadata.catalog.type.ChangeDescription;
 import org.openmetadata.catalog.type.Column;
 import org.openmetadata.catalog.type.ColumnJoin;
@@ -65,7 +65,6 @@
 import org.openmetadata.catalog.type.TableJoins;
 import org.openmetadata.catalog.type.TableProfile;
 import org.openmetadata.catalog.type.TagLabel;
-import org.openmetadata.catalog.type.TestCaseResult;
 import org.openmetadata.catalog.util.EntityInterface;
 import org.openmetadata.catalog.util.EntityUtil;
 import org.openmetadata.catalog.util.EntityUtil.Fields;
@@ -268,17 +267,17 @@ public Table addTableTest(UUID tableId, TableTest tableTest) throws IOException,
         && tableTest.getResults() != null
         && !tableTest.getResults().isEmpty()) {
       TableTest prevTableTest = storedMapTableTests.get(tableTest.getId());
-      List<TestCaseResult> prevTestCaseResults = prevTableTest.getResults();
-      List<TestCaseResult> newTestCaseResults = tableTest.getResults();
+      List<Object> prevTestCaseResults = prevTableTest.getResults();
+      List<Object> newTestCaseResults = tableTest.getResults();
       newTestCaseResults.addAll(prevTestCaseResults);
       tableTest.setResults(newTestCaseResults);
     }
 
     storedMapTableTests.put(tableTest.getId(), tableTest);
-    List<TableTest> updatedQueries = new ArrayList<>(storedMapTableTests.values());
+    List<TableTest> updatedTests = new ArrayList<>(storedMapTableTests.values());
     daoCollection
         .entityExtensionDAO()
-        .insert(tableId.toString(), "table.tableTests", "tableTest", JsonUtils.pojoToJson(updatedQueries));
+        .insert(tableId.toString(), "table.tableTests", "tableTest", JsonUtils.pojoToJson(updatedTests));
     setFields(table, Fields.EMPTY_FIELDS);
     return table.withTableTests(getTableTests(table));
   }

diff --git a/...est-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java b/...est-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java
@@ -52,12 +52,12 @@
 import org.openmetadata.catalog.Entity;
 import org.openmetadata.catalog.api.data.CreateTable;
 import org.openmetadata.catalog.entity.data.Table;
-import org.openmetadata.catalog.entity.data.TableTest;
 import org.openmetadata.catalog.jdbi3.CollectionDAO;
 import org.openmetadata.catalog.jdbi3.TableRepository;
 import org.openmetadata.catalog.resources.Collection;
 import org.openmetadata.catalog.security.Authorizer;
 import org.openmetadata.catalog.security.SecurityUtil;
+import org.openmetadata.catalog.tests.TableTest;
 import org.openmetadata.catalog.type.DataModel;
 import org.openmetadata.catalog.type.EntityHistory;
 import org.openmetadata.catalog.type.Include;

diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json
@@ -179,6 +179,14 @@
             "$ref": "#/definitions/column"
           },
           "default": null
+        },
+        "columnTests": {
+          "description": "List of column test cases that ran against a table column.",
+          "type": "array",
+          "items": {
+            "$ref": "../../tests/columnTest.json"
+          },
+          "default": null
         }
       },
       "required": ["name", "dataType"],
@@ -592,7 +600,7 @@
       "description": "List of test cases that ran against a table.",
       "type": "array",
       "items": {
-        "$ref": "../../test/tableTest.json"
+        "$ref": "../../tests/tableTest.json"
       },
       "default": null
     },

diff --git a/catalog-rest-service/src/main/resources/json/schema/test/columnTest.json b/catalog-rest-service/src/main/resources/json/schema/test/columnTest.json