From 883e88322b229b01a790c12a37fd2ee2ab51b26e Mon Sep 17 00:00:00 2001 From: Leiqing Cai Date: Sun, 20 Sep 2020 22:33:11 -0700 Subject: [PATCH] Support CREATE TABLE verification Rewrite the target table of the CREATE TABLE statement, run both control and test queries, run SHOW CREATE TABLE query as the check. --- .../framework/CreateTableVerification.java | 90 ++++++++++++ .../presto/verifier/framework/QueryType.java | 2 + .../framework/VerificationFactory.java | 9 ++ .../verifier/rewrite/QueryRewriter.java | 15 ++ .../TestCreateTableVerification.java | 128 ++++++++++++++++++ .../framework/TestVerificationManager.java | 2 +- 6 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/framework/CreateTableVerification.java create mode 100644 presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestCreateTableVerification.java diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/CreateTableVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/CreateTableVerification.java new file mode 100644 index 000000000000..e9dc94b1044d --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/CreateTableVerification.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.framework; + +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.tree.CreateTable; +import com.facebook.presto.sql.tree.QualifiedName; +import com.facebook.presto.sql.tree.ShowCreate; +import com.facebook.presto.sql.tree.Statement; +import com.facebook.presto.verifier.prestoaction.PrestoAction.ResultSetConverter; +import com.facebook.presto.verifier.prestoaction.QueryActions; +import com.facebook.presto.verifier.prestoaction.SqlExceptionClassifier; +import com.facebook.presto.verifier.rewrite.QueryRewriter; + +import java.sql.SQLException; +import java.util.Objects; +import java.util.Optional; + +import static com.facebook.presto.sql.tree.ShowCreate.Type.TABLE; +import static java.util.Objects.requireNonNull; + +public class CreateTableVerification + extends DdlVerification +{ + public static final ResultSetConverter SHOW_CREATE_TABLE_CONVERTER = resultSet -> { + try { + return Optional.of(resultSet.getString("Create Table")); + } + catch (SQLException e) { + throw new RuntimeException(e); + } + }; + private static final QualifiedName DUMMY_TABLE_NAME = QualifiedName.of("dummy"); + + private final QueryRewriter queryRewriter; + + public CreateTableVerification( + SqlParser sqlParser, + QueryActions queryActions, + SourceQuery sourceQuery, + QueryRewriter queryRewriter, + SqlExceptionClassifier exceptionClassifier, + VerificationContext verificationContext, + VerifierConfig verifierConfig) + { + super(sqlParser, queryActions, sourceQuery, exceptionClassifier, verificationContext, verifierConfig, SHOW_CREATE_TABLE_CONVERTER); + this.queryRewriter = requireNonNull(queryRewriter, "queryRewriter is null"); + } + + @Override + protected QueryObjectBundle getQueryRewrite(ClusterType clusterType) + { + return queryRewriter.rewriteQuery(getSourceQuery().getQuery(clusterType), clusterType); + } + + @Override + protected Statement getChecksumQuery(QueryObjectBundle queryBundle) + { + return new ShowCreate(TABLE, queryBundle.getObjectName()); + } + + @Override + protected boolean match(CreateTable controlObject, CreateTable testObject, QueryObjectBundle control, QueryObjectBundle test) + { + controlObject = new CreateTable( + DUMMY_TABLE_NAME, + controlObject.getElements(), + controlObject.isNotExists(), + controlObject.getProperties(), + controlObject.getComment()); + testObject = new CreateTable( + DUMMY_TABLE_NAME, + testObject.getElements(), + testObject.isNotExists(), + testObject.getProperties(), + testObject.getComment()); + return Objects.equals(controlObject, testObject); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryType.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryType.java index b151989a6b13..2284e81bf4ee 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryType.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryType.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateTableAsSelect; import com.facebook.presto.sql.tree.CreateView; import com.facebook.presto.sql.tree.Insert; @@ -27,6 +28,7 @@ public enum QueryType INSERT(Insert.class), QUERY(Query.class), CREATE_VIEW(CreateView.class), + CREATE_TABLE(CreateTable.class), UNSUPPORTED(); private final Optional> statementClass; diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationFactory.java index ffdead851222..70faa1984cb7 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationFactory.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationFactory.java @@ -119,6 +119,15 @@ public Verification get(SourceQuery sourceQuery, Optional e exceptionClassifier, verificationContext, verifierConfig); + case CREATE_TABLE: + return new CreateTableVerification( + sqlParser, + queryActions, + sourceQuery, + queryRewriter, + exceptionClassifier, + verificationContext, + verifierConfig); default: throw new IllegalStateException(format("Unsupported query type: %s", queryType)); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java index 3776f5c777a8..25e3056b63e1 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java @@ -207,6 +207,21 @@ public QueryObjectBundle rewriteQuery(@Language("SQL") String query, ClusterType ImmutableList.of(new DropView(temporaryViewName, true)), clusterType); } + if (statement instanceof CreateTable) { + CreateTable createTable = (CreateTable) statement; + QualifiedName temporaryTableName = generateTemporaryName(Optional.empty(), prefix); + return new QueryObjectBundle( + temporaryTableName, + ImmutableList.of(), + new CreateTable( + temporaryTableName, + createTable.getElements(), + createTable.isNotExists(), + applyPropertyOverride(createTable.getProperties(), properties), + createTable.getComment()), + ImmutableList.of(new DropTable(temporaryTableName, true)), + clusterType); + } throw new IllegalStateException(format("Unsupported query type: %s", statement.getClass())); } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestCreateTableVerification.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestCreateTableVerification.java new file mode 100644 index 000000000000..e1cfdfa397b0 --- /dev/null +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestCreateTableVerification.java @@ -0,0 +1,128 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.framework; + +import com.facebook.presto.verifier.event.VerifierQueryEvent; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.FAILED; +import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SKIPPED; +import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SUCCEEDED; +import static com.facebook.presto.verifier.framework.DdlMatchResult.MatchType.CONTROL_NOT_PARSABLE; +import static com.facebook.presto.verifier.framework.DdlMatchResult.MatchType.MATCH; +import static com.facebook.presto.verifier.framework.DdlMatchResult.MatchType.MISMATCH; +import static com.facebook.presto.verifier.framework.DdlMatchResult.MatchType.TEST_NOT_PARSABLE; +import static org.testng.Assert.assertTrue; + +@Test(singleThreaded = true) +public class TestCreateTableVerification + extends AbstractDdlVerificationTest +{ + public TestCreateTableVerification() + throws Exception + { + } + + @Test + public void testSucceeded() + { + String query = "CREATE TABLE succeeded (x int, ds varchar) COMMENT 'test table'"; + + Optional event = runVerification(query, query); + assertTrue(event.isPresent()); + assertEvent(event.get(), SUCCEEDED, Optional.of(MATCH), false); + + getQueryRunner().execute("CREATE TABLE like_table (x int, ds varchar)"); + query = "CREATE TABLE succeeded (LIKE like_table INCLUDING PROPERTIES)"; + + event = runVerification(query, query); + assertTrue(event.isPresent()); + System.out.println(event.get().getErrorMessage()); + assertEvent(event.get(), SUCCEEDED, Optional.of(MATCH), false); + + getQueryRunner().execute("DROP TABLE like_table"); + } + + @Test + public void testSucceededExists() + { + getQueryRunner().execute("CREATE TABLE succeeded_exists (x int, ds varchar)"); + String query = "CREATE TABLE IF NOT EXISTS succeeded_exists (x int, ds varchar)"; + + Optional event = runVerification(query, query); + assertTrue(event.isPresent()); + assertEvent(event.get(), SUCCEEDED, Optional.of(MATCH), false); + + getQueryRunner().execute("DROP TABLE succeeded_exists"); + } + + @Test + public void testControlNotParsable() + { + String query = "CREATE TABLE control_not_parsable (x int, ds varchar) COMMENT 'test table'"; + MockPrestoAction prestoAction = new MockPrestoAction(ImmutableMap.of(1, "CREATE TABLE succeeded (x int, ds varchar) 'test table'")); + + Optional event = verify(getSourceQuery(query, query), false, prestoAction); + assertTrue(event.isPresent()); + System.out.println(event.get().getErrorMessage()); + assertEvent(event.get(), FAILED, Optional.of(CONTROL_NOT_PARSABLE), false); + } + + @Test + public void testTestNotParsable() + { + String query = "CREATE TABLE test_not_parsable (x int, ds varchar) COMMENT 'test table'"; + MockPrestoAction prestoAction = new MockPrestoAction(ImmutableMap.of(2, "CREATE TABLE test_not_parsable (x int, ds varchar) 'test table'")); + + Optional event = verify(getSourceQuery(query, query), false, prestoAction); + assertTrue(event.isPresent()); + assertEvent(event.get(), FAILED, Optional.of(TEST_NOT_PARSABLE), false); + } + + @Test + public void testMismatched() + { + String query = "CREATE TABLE mismatched (x int, ds varchar)"; + MockPrestoAction prestoAction = new MockPrestoAction(ImmutableMap.of( + 1, "CREATE TABLE mismatched (x int, ds varchar)", + 2, "CREATE TABLE mismatched (ds varchar)")); + + Optional event = verify(getSourceQuery(query, query), false, prestoAction); + assertTrue(event.isPresent()); + assertEvent(event.get(), FAILED, Optional.of(MISMATCH), false); + } + + @Test + public void testSkipped() + { + Optional event = runVerification( + "CREATE TABLE failed (LIKE non_existing)", + "CREATE TABLE failed (LIKE non_existing)"); + assertTrue(event.isPresent()); + assertEvent(event.get(), SKIPPED, Optional.empty(), false); + } + + @Test + public void testFailed() + { + Optional event = runVerification( + "CREATE TABLE failed (x int, ds varchar)", + "CREATE TABLE failed (LIKE non_existing)"); + assertTrue(event.isPresent()); + assertEvent(event.get(), FAILED, Optional.empty(), false); + } +} diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java index 43183e7ad1d9..dfb1501e4728 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java @@ -155,7 +155,7 @@ public void testFilters() List queries = ImmutableList.of( createSourceQuery("q1", "CREATE TABLE t1 (x int)", "CREATE TABLE t1 (x int)"), createSourceQuery("q2", "CREATE TABLE t1 (x int)", "CREATE TABLE t1 (x int)"), - createSourceQuery("q3", "CREATE TABLE t1 (x int)", "CREATE TABLE t1 (x int)"), + createSourceQuery("q3", "SHOW TABLES", "SHOW TABLES"), createSourceQuery("q4", "SHOW FUNCTIONS", "SHOW FUNCTIONS"), createSourceQuery("q5", "SELECT * FROM t1", "INSERT INTO t2 SELECT * FROM t1"), createSourceQuery("q6", "SELECT * FROM t1", "SELECT FROM t1"));