-
Notifications
You must be signed in to change notification settings - Fork 2.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add azure native-fs product tests for table formats
- Loading branch information
1 parent
7716347
commit c7a2a6a
Showing
8 changed files
with
288 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
6 changes: 6 additions & 0 deletions
6
...n/resources/docker/presto-product-tests/conf/environment/multinode-azure/delta.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| connector.name=delta_lake | ||
| hive.metastore.uri=thrift://hadoop-master:9083 | ||
| fs.hadoop.enabled=false | ||
| fs.native-azure.enabled=true | ||
| azure.auth-type=ACCESS_KEY | ||
| azure.access-key=${ENV:ABFS_ACCESS_KEY} |
7 changes: 7 additions & 0 deletions
7
...resources/docker/presto-product-tests/conf/environment/multinode-azure/iceberg.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| connector.name=iceberg | ||
| hive.metastore.uri=thrift://hadoop-master:9083 | ||
| iceberg.file-format=PARQUET | ||
| fs.hadoop.enabled=false | ||
| fs.native-azure.enabled=true | ||
| azure.auth-type=ACCESS_KEY | ||
| azure.access-key=${ENV:ABFS_ACCESS_KEY} |
82 changes: 82 additions & 0 deletions
82
...rino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAzure.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| /* | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package io.trino.tests.product.deltalake; | ||
|
|
||
| import com.google.inject.Inject; | ||
| import com.google.inject.name.Named; | ||
| import io.trino.tempto.BeforeMethodWithContext; | ||
| import io.trino.tempto.ProductTest; | ||
| import org.testng.annotations.Test; | ||
|
|
||
| import static io.trino.tempto.assertions.QueryAssert.Row.row; | ||
| import static io.trino.testing.TestingNames.randomNameSuffix; | ||
| import static io.trino.tests.product.TestGroups.DELTA_LAKE_AZURE; | ||
| import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; | ||
| import static io.trino.tests.product.utils.QueryExecutors.onTrino; | ||
| import static java.lang.String.format; | ||
| import static java.util.Objects.requireNonNull; | ||
| import static org.assertj.core.api.Assertions.assertThat; | ||
|
|
||
| public class TestDeltaLakeAzure | ||
| extends ProductTest | ||
| { | ||
| @Inject | ||
| @Named("databases.presto.abfs_schema") | ||
| private String schema; | ||
| private String schemaLocation; | ||
|
|
||
| @BeforeMethodWithContext | ||
| public void setUp() | ||
| { | ||
| String container = requireNonNull(System.getenv("ABFS_CONTAINER"), "Environment variable not set: ABFS_CONTAINER"); | ||
| String account = requireNonNull(System.getenv("ABFS_ACCOUNT"), "Environment variable not set: ABFS_ACCOUNT"); | ||
| schemaLocation = format("abfs://%s@%s.dfs.core.windows.net/%s", container, account, schema); | ||
| } | ||
|
|
||
| @Test(groups = {DELTA_LAKE_AZURE, PROFILE_SPECIFIC_TESTS}) | ||
| public void testCreateAndSelectNationTable() | ||
| { | ||
| String tableName = "nation_" + randomNameSuffix(); | ||
| String tableLocation = schemaLocation + "/" + tableName; | ||
| onTrino().executeQuery(format( | ||
| "CREATE TABLE delta.default.%1$s WITH (location = '%2$s/%1$s') AS SELECT * FROM tpch.tiny.nation", | ||
| tableName, | ||
| tableLocation)); | ||
|
|
||
| assertThat(onTrino().executeQuery("SELECT count(*) FROM delta.default." + tableName)).containsOnly(row(25)); | ||
| onTrino().executeQuery("DROP TABLE delta.default." + tableName); | ||
| } | ||
|
|
||
| @Test(groups = {DELTA_LAKE_AZURE, PROFILE_SPECIFIC_TESTS}) | ||
| public void testBasicWriteOperations() | ||
| { | ||
| String tableName = "table_write_operations_" + randomNameSuffix(); | ||
| String tableLocation = schemaLocation + "/" + tableName; | ||
|
|
||
| onTrino().executeQuery(format( | ||
| "CREATE TABLE delta.default.%1$s (a_bigint bigint, a_varchar varchar) WITH (location = '%2$s/%1$s')", | ||
| tableName, | ||
| tableLocation)); | ||
|
|
||
| onTrino().executeQuery(format("INSERT INTO delta.default.%s VALUES (1, 'hello world')".formatted(tableName))); | ||
| assertThat(onTrino().executeQuery("SELECT * FROM delta.default." + tableName)).containsOnly(row(1L, "hello world")); | ||
|
|
||
| onTrino().executeQuery(format("UPDATE delta.default.%s SET a_varchar = 'hallo Welt' WHERE a_bigint = 1".formatted(tableName))); | ||
| assertThat(onTrino().executeQuery("SELECT * FROM delta.default." + tableName)).containsOnly(row(1L, "hallo Welt")); | ||
|
|
||
| onTrino().executeQuery(format("DELETE FROM delta.default.%s WHERE a_bigint = 1".formatted(tableName))); | ||
| assertThat(onTrino().executeQuery("SELECT * FROM delta.default." + tableName)).hasNoRows(); | ||
| onTrino().executeQuery("DROP TABLE delta.default." + tableName); | ||
| } | ||
| } |
91 changes: 91 additions & 0 deletions
91
testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveAzure.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| /* | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package io.trino.tests.product.hive; | ||
|
|
||
| import com.google.inject.Inject; | ||
| import com.google.inject.name.Named; | ||
| import io.trino.tempto.AfterMethodWithContext; | ||
| import io.trino.tempto.BeforeMethodWithContext; | ||
| import io.trino.tempto.ProductTest; | ||
| import org.testng.annotations.Test; | ||
|
|
||
| import static io.trino.tempto.assertions.QueryAssert.Row.row; | ||
| import static io.trino.testing.TestingNames.randomNameSuffix; | ||
| import static io.trino.tests.product.TestGroups.HIVE_AZURE; | ||
| import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; | ||
| import static io.trino.tests.product.utils.QueryExecutors.onHive; | ||
| import static io.trino.tests.product.utils.QueryExecutors.onTrino; | ||
| import static java.lang.String.format; | ||
| import static java.util.Objects.requireNonNull; | ||
| import static org.assertj.core.api.Assertions.assertThat; | ||
|
|
||
| public class TestHiveAzure | ||
| extends ProductTest | ||
| { | ||
| @Inject | ||
| @Named("databases.presto.abfs_schema") | ||
| private String schema; | ||
| private String schemaLocation; | ||
|
|
||
| @BeforeMethodWithContext | ||
| public void setUp() | ||
| { | ||
| String container = requireNonNull(System.getenv("ABFS_CONTAINER"), "Environment variable not set: ABFS_CONTAINER"); | ||
| String account = requireNonNull(System.getenv("ABFS_ACCOUNT"), "Environment variable not set: ABFS_ACCOUNT"); | ||
| schemaLocation = format("abfs://%s@%s.dfs.core.windows.net/%s", container, account, schema); | ||
|
|
||
| onHive().executeQuery("dfs -rm -f -r " + schemaLocation); | ||
| onHive().executeQuery("dfs -mkdir -p " + schemaLocation); | ||
| } | ||
|
|
||
| @AfterMethodWithContext | ||
| public void tearDown() | ||
| { | ||
| onHive().executeQuery("dfs -mkdir -p " + schemaLocation); | ||
| } | ||
|
|
||
| @Test(groups = {HIVE_AZURE, PROFILE_SPECIFIC_TESTS}) | ||
| public void testCreateAndSelectNationTable() | ||
| { | ||
| String tableName = "nation_" + randomNameSuffix(); | ||
|
|
||
| try { | ||
| onTrino().executeQuery(format("CREATE SCHEMA hive.az WITH (location = '%1$s')", schemaLocation)); | ||
| onTrino().executeQuery(format("CREATE TABLE hive.az.%1$s AS SELECT * FROM tpch.tiny.nation", tableName)); | ||
|
|
||
| assertThat(onTrino().executeQuery("SELECT count(*) FROM hive.az." + tableName)).containsOnly(row(25)); | ||
| } | ||
| finally { | ||
| onTrino().executeQuery("DROP TABLE hive.az." + tableName); | ||
| onTrino().executeQuery("DROP SCHEMA hive.az"); | ||
| } | ||
| } | ||
|
|
||
| @Test(groups = {HIVE_AZURE, PROFILE_SPECIFIC_TESTS}) | ||
| public void testBasicWriteOperations() | ||
| { | ||
| String tableName = "table_write_operations_" + randomNameSuffix(); | ||
| try { | ||
| onTrino().executeQuery(format("CREATE SCHEMA hive.az WITH (location = '%1$s')", schemaLocation)); | ||
| onTrino().executeQuery(format("CREATE TABLE hive.az.%1$s (a_bigint bigint, a_varchar varchar)", tableName)); | ||
|
|
||
| onTrino().executeQuery(format("INSERT INTO hive.az.%s VALUES (1, 'hello world')".formatted(tableName))); | ||
| assertThat(onTrino().executeQuery("SELECT * FROM hive.az." + tableName)).containsOnly(row(1L, "hello world")); | ||
| } | ||
| finally { | ||
| onTrino().executeQuery("DROP TABLE hive.az." + tableName); | ||
| onTrino().executeQuery("DROP SCHEMA hive.az"); | ||
| } | ||
| } | ||
| } |
82 changes: 82 additions & 0 deletions
82
...ng/trino-product-tests/src/main/java/io/trino/tests/product/iceberg/TestIcebergAzure.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| /* | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package io.trino.tests.product.iceberg; | ||
|
|
||
| import com.google.inject.Inject; | ||
| import com.google.inject.name.Named; | ||
| import io.trino.tempto.BeforeMethodWithContext; | ||
| import io.trino.tempto.ProductTest; | ||
| import org.testng.annotations.Test; | ||
|
|
||
| import static io.trino.tempto.assertions.QueryAssert.Row.row; | ||
| import static io.trino.testing.TestingNames.randomNameSuffix; | ||
| import static io.trino.tests.product.TestGroups.ICEBERG_AZURE; | ||
| import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; | ||
| import static io.trino.tests.product.utils.QueryExecutors.onTrino; | ||
| import static java.lang.String.format; | ||
| import static java.util.Objects.requireNonNull; | ||
| import static org.assertj.core.api.Assertions.assertThat; | ||
|
|
||
| public class TestIcebergAzure | ||
| extends ProductTest | ||
| { | ||
| @Inject | ||
| @Named("databases.presto.abfs_schema") | ||
| private String schema; | ||
| private String schemaLocation; | ||
|
|
||
| @BeforeMethodWithContext | ||
| public void setUp() | ||
| { | ||
| String container = requireNonNull(System.getenv("ABFS_CONTAINER"), "Environment variable not set: ABFS_CONTAINER"); | ||
| String account = requireNonNull(System.getenv("ABFS_ACCOUNT"), "Environment variable not set: ABFS_ACCOUNT"); | ||
| schemaLocation = format("abfs://%s@%s.dfs.core.windows.net/%s", container, account, schema); | ||
| } | ||
|
|
||
| @Test(groups = {ICEBERG_AZURE, PROFILE_SPECIFIC_TESTS}) | ||
| public void testCreateAndSelectNationTable() | ||
| { | ||
| String tableName = "nation_" + randomNameSuffix(); | ||
| String tableLocation = schemaLocation + "/" + tableName; | ||
| onTrino().executeQuery(format( | ||
| "CREATE TABLE iceberg.default.%1$s WITH (location = '%2$s/%1$s') AS SELECT * FROM tpch.tiny.nation", | ||
| tableName, | ||
| tableLocation)); | ||
|
|
||
| assertThat(onTrino().executeQuery("SELECT count(*) FROM iceberg.default." + tableName)).containsOnly(row(25)); | ||
| onTrino().executeQuery("DROP TABLE iceberg.default." + tableName); | ||
| } | ||
|
|
||
| @Test(groups = {ICEBERG_AZURE, PROFILE_SPECIFIC_TESTS}) | ||
| public void testBasicWriteOperations() | ||
| { | ||
| String tableName = "table_write_operations_" + randomNameSuffix(); | ||
| String tableLocation = schemaLocation + "/" + tableName; | ||
|
|
||
| onTrino().executeQuery(format( | ||
| "CREATE TABLE iceberg.default.%1$s (a_bigint bigint, a_varchar varchar) WITH (location = '%2$s/%1$s')", | ||
| tableName, | ||
| tableLocation)); | ||
|
|
||
| onTrino().executeQuery(format("INSERT INTO iceberg.default.%s VALUES (1, 'hello world')".formatted(tableName))); | ||
| assertThat(onTrino().executeQuery("SELECT * FROM iceberg.default." + tableName)).containsOnly(row(1L, "hello world")); | ||
|
|
||
| onTrino().executeQuery(format("UPDATE iceberg.default.%s SET a_varchar = 'hallo Welt' WHERE a_bigint = 1".formatted(tableName))); | ||
| assertThat(onTrino().executeQuery("SELECT * FROM iceberg.default." + tableName)).containsOnly(row(1L, "hallo Welt")); | ||
|
|
||
| onTrino().executeQuery(format("DELETE FROM iceberg.default.%s WHERE a_bigint = 1".formatted(tableName))); | ||
| assertThat(onTrino().executeQuery("SELECT * FROM iceberg.default." + tableName)).hasNoRows(); | ||
| onTrino().executeQuery("DROP TABLE iceberg.default." + tableName); | ||
| } | ||
| } |