-
Notifications
You must be signed in to change notification settings - Fork 308
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
bd1e353
commit 6293162
Showing
12 changed files
with
215 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
52 changes: 52 additions & 0 deletions
52
warehouse/integrations/datalake/testdata/conf/metastore-site.xml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| <configuration> | ||
| <property> | ||
| <name>metastore.thrift.uris</name> | ||
| <value>thrift://hive-metastore:9083</value> | ||
| <description>Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore.</description> | ||
| </property> | ||
| <property> | ||
| <name>metastore.task.threads.always</name> | ||
| <value>org.apache.hadoop.hive.metastore.events.EventCleanerTask,org.apache.hadoop.hive.metastore.MaterializationsCacheCleanerTask</value> | ||
| </property> | ||
| <property> | ||
| <name>metastore.expression.proxy</name> | ||
| <value>org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy</value> | ||
| </property> | ||
| <property> | ||
| <name>javax.jdo.option.ConnectionDriverName</name> | ||
| <value>com.mysql.jdbc.Driver</value> | ||
| </property> | ||
| <property> | ||
| <name>javax.jdo.option.ConnectionURL</name> | ||
| <value>jdbc:mysql://mariadb:3306/metastore_db</value> | ||
| </property> | ||
| <property> | ||
| <name>javax.jdo.option.ConnectionUserName</name> | ||
| <value>admin</value> | ||
| </property> | ||
| <property> | ||
| <name>javax.jdo.option.ConnectionPassword</name> | ||
| <value>admin</value> | ||
| </property> | ||
|
|
||
| <property> | ||
| <name>fs.s3a.impl</name> | ||
| <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value> | ||
| </property> | ||
| <property> | ||
| <name>fs.s3a.access.key</name> | ||
| <value>MYACCESSKEY</value> | ||
| </property> | ||
| <property> | ||
| <name>fs.s3a.secret.key</name> | ||
| <value>MYSECRETKEY</value> | ||
| </property> | ||
| <property> | ||
| <name>fs.s3a.endpoint</name> | ||
| <value>http://minio:9000</value> | ||
| </property> | ||
| <property> | ||
| <name>fs.s3a.path.style.access</name> | ||
| <value>true</value> | ||
| </property> | ||
| </configuration> |
29 changes: 29 additions & 0 deletions
29
warehouse/integrations/datalake/testdata/docker-compose.trino.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| version: '3.7' | ||
| services: | ||
| trino: | ||
| image: trinodb/trino:362 | ||
| ports: | ||
| - 8080 | ||
| volumes: | ||
| - ./etc:/etc/trino/:ro | ||
|
|
||
| mariadb: | ||
| image: mariadb:10.5.8 | ||
| ports: | ||
| - 3306 | ||
| environment: | ||
| MYSQL_ROOT_PASSWORD: admin | ||
| MYSQL_USER: admin | ||
| MYSQL_PASSWORD: admin | ||
| MYSQL_DATABASE: metastore_db | ||
|
|
||
| hive-metastore: | ||
| image: bitsondatadev/hive-metastore:latest | ||
| ports: | ||
| - 9083 | ||
| volumes: | ||
| - ./conf/metastore-site.xml:/opt/apache-hive-metastore-3.0.0-bin/conf/metastore-site.xml:ro | ||
| environment: | ||
| METASTORE_DB_HOSTNAME: mariadb | ||
| depends_on: | ||
| - mariadb |
9 changes: 9 additions & 0 deletions
9
warehouse/integrations/datalake/testdata/etc/catalog/minio.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| connector.name=hive-hadoop2 | ||
| hive.metastore.uri=thrift://hive-metastore:9083 | ||
| hive.s3.path-style-access=true | ||
| hive.s3.endpoint=http://minio:9000 | ||
| hive.s3.aws-access-key=MYACCESSKEY | ||
| hive.s3.aws-secret-key=MYSECRETKEY | ||
| hive.non-managed-table-writes-enabled=true | ||
| hive.s3select-pushdown.enabled=true | ||
| hive.storage-format=ORC |
2 changes: 2 additions & 0 deletions
2
warehouse/integrations/datalake/testdata/etc/catalog/tpcds.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| connector.name=tpcds | ||
| tpcds.splits-per-node=4 |
2 changes: 2 additions & 0 deletions
2
warehouse/integrations/datalake/testdata/etc/catalog/tpch.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| connector.name=tpch | ||
| tpch.splits-per-node=4 |
11 changes: 11 additions & 0 deletions
11
warehouse/integrations/datalake/testdata/etc/config.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| #single node install config | ||
| coordinator=true | ||
| node-scheduler.include-coordinator=true | ||
| http-server.http.port=8080 | ||
| discovery-server.enabled=true | ||
| discovery.uri=http://localhost:8080 | ||
|
|
||
| # needed to treat https requests over http as secure (in katakoda) | ||
| # https://github.com/prestosql/presto/pull/1442 | ||
| # https://github.com/prestosql/presto/pull/3714 | ||
| http-server.process-forwarded=true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| -server | ||
| -Xmx1G | ||
| -XX:-UseBiasedLocking | ||
| -XX:+UseG1GC | ||
| -XX:G1HeapRegionSize=32M | ||
| -XX:+ExplicitGCInvokesConcurrent | ||
| -XX:+HeapDumpOnOutOfMemoryError | ||
| -XX:+UseGCOverheadLimit | ||
| -XX:+ExitOnOutOfMemoryError | ||
| -XX:ReservedCodeCacheSize=256M | ||
| -Djdk.attach.allowAttachSelf=true | ||
| -Djdk.nio.maxCachedBufferSize=2000000 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| # Enable verbose logging from Trino | ||
| io.trino=DEBUG |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| node.environment=docker | ||
| node.data-dir=/data/trino | ||
| plugin.dir=/usr/lib/trino/plugin |