Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Added default LogStore implementation for the .gs scheme
(Cherry-pick of #1071)

This PR adds gs scheme to default LogStore implementation
Resolves #1068

Unit Test and manul testing with GCS (details [here](#1071 (comment)))

Yes, users can automatically derive the GCS log store conf based on the path

Closes #1071

Signed-off-by: Venki Korukanti <venki.korukanti@gmail.com>
GitOrigin-RevId: 5e877a8a9723bc5cae253cd27924da4aeadcb7c8
  • Loading branch information
ahlag authored and vkorukanti committed Apr 20, 2022
1 parent a4c0107 commit 7db19f6
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 0 deletions.
Expand Up @@ -153,10 +153,12 @@ object DelegatingLogStore {
val defaultS3LogStoreClassName = classOf[io.delta.storage.S3SingleDriverLogStore].getName
val defaultAzureLogStoreClassName = classOf[io.delta.storage.AzureLogStore].getName
val defaultHDFSLogStoreClassName = classOf[io.delta.storage.HDFSLogStore].getName
val defaultGCSLogStoreClassName = classOf[io.delta.storage.GCSLogStore].getName

// Supported schemes with default.
val s3Schemes = Set("s3", "s3a", "s3n")
val azureSchemes = Set("abfs", "abfss", "adl", "wasb", "wasbs")
val gsSchemes = Set("gs")

// Returns the default LogStore class name for `scheme`.
// None if we do not have a default for it.
Expand All @@ -165,6 +167,8 @@ object DelegatingLogStore {
return Some(defaultS3LogStoreClassName)
} else if (DelegatingLogStore.azureSchemes(scheme: String)) {
return Some(defaultAzureLogStoreClassName)
} else if (DelegatingLogStore.gsSchemes(scheme: String)) {
return Some(defaultGCSLogStoreClassName)
}
None
}
Expand Down
Expand Up @@ -120,6 +120,13 @@ class DelegatingLogStoreSuite
expClassName = DelegatingLogStore.defaultAzureLogStoreClassName,
expAdaptor = true)
}
for (scheme <- DelegatingLogStore.gsSchemes) {
testDelegatingLogStore(
scheme,
schemeConf = None,
expClassName = DelegatingLogStore.defaultGCSLogStoreClassName,
expAdaptor = true)
}
testDelegatingLogStore(
scheme = fakeSchemeWithNoDefault,
schemeConf = None,
Expand Down

0 comments on commit 7db19f6

Please sign in to comment.