From 7db19f6823e3f73bf68c936c9915007f67432227 Mon Sep 17 00:00:00 2001 From: Chang Yong Lik Date: Mon, 18 Apr 2022 21:27:06 -0700 Subject: [PATCH] Added default LogStore implementation for the .gs scheme (Cherry-pick of https://github.com/delta-io/delta/pull/1071) This PR adds gs scheme to default LogStore implementation Resolves https://github.com/delta-io/delta/issues/1068 Unit Test and manul testing with GCS (details [here](https://github.com/delta-io/delta/pull/1071#issuecomment-1101812886)) Yes, users can automatically derive the GCS log store conf based on the path Closes delta-io/delta#1071 Signed-off-by: Venki Korukanti GitOrigin-RevId: 5e877a8a9723bc5cae253cd27924da4aeadcb7c8 --- .../spark/sql/delta/storage/DelegatingLogStore.scala | 4 ++++ .../apache/spark/sql/delta/DelegatingLogStoreSuite.scala | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/sql/delta/storage/DelegatingLogStore.scala b/core/src/main/scala/org/apache/spark/sql/delta/storage/DelegatingLogStore.scala index df5a75f173..ea5571674e 100644 --- a/core/src/main/scala/org/apache/spark/sql/delta/storage/DelegatingLogStore.scala +++ b/core/src/main/scala/org/apache/spark/sql/delta/storage/DelegatingLogStore.scala @@ -153,10 +153,12 @@ object DelegatingLogStore { val defaultS3LogStoreClassName = classOf[io.delta.storage.S3SingleDriverLogStore].getName val defaultAzureLogStoreClassName = classOf[io.delta.storage.AzureLogStore].getName val defaultHDFSLogStoreClassName = classOf[io.delta.storage.HDFSLogStore].getName + val defaultGCSLogStoreClassName = classOf[io.delta.storage.GCSLogStore].getName // Supported schemes with default. val s3Schemes = Set("s3", "s3a", "s3n") val azureSchemes = Set("abfs", "abfss", "adl", "wasb", "wasbs") + val gsSchemes = Set("gs") // Returns the default LogStore class name for `scheme`. // None if we do not have a default for it. @@ -165,6 +167,8 @@ object DelegatingLogStore { return Some(defaultS3LogStoreClassName) } else if (DelegatingLogStore.azureSchemes(scheme: String)) { return Some(defaultAzureLogStoreClassName) + } else if (DelegatingLogStore.gsSchemes(scheme: String)) { + return Some(defaultGCSLogStoreClassName) } None } diff --git a/core/src/test/scala/org/apache/spark/sql/delta/DelegatingLogStoreSuite.scala b/core/src/test/scala/org/apache/spark/sql/delta/DelegatingLogStoreSuite.scala index 375b0242b8..7fc4c4ede9 100644 --- a/core/src/test/scala/org/apache/spark/sql/delta/DelegatingLogStoreSuite.scala +++ b/core/src/test/scala/org/apache/spark/sql/delta/DelegatingLogStoreSuite.scala @@ -120,6 +120,13 @@ class DelegatingLogStoreSuite expClassName = DelegatingLogStore.defaultAzureLogStoreClassName, expAdaptor = true) } + for (scheme <- DelegatingLogStore.gsSchemes) { + testDelegatingLogStore( + scheme, + schemeConf = None, + expClassName = DelegatingLogStore.defaultGCSLogStoreClassName, + expAdaptor = true) + } testDelegatingLogStore( scheme = fakeSchemeWithNoDefault, schemeConf = None,