Skip to content

Commit

Permalink
Upgrade delta-sharing-client to 1.0.5 (#2955)
Browse files Browse the repository at this point in the history
<!--
Thanks for sending a pull request!  Here are some tips for you:
1. If this is your first time, please read our contributor guidelines:
https://github.com/delta-io/delta/blob/master/CONTRIBUTING.md
2. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP]
Your PR title ...'.
  3. Be sure to keep the PR description updated to reflect all changes.
  4. Please write your PR title to summarize what this PR proposes.
5. If possible, provide a concise example to reproduce the issue for a
faster review.
6. If applicable, include the corresponding issue number in the PR title
and link it in the body.
-->

#### Which Delta project/connector is this regarding?
<!--
Please add the component selected below to the beginning of the pull
request title
For example: [Spark] Title of my pull request
-->

- [ ] Spark
- [ ] Standalone
- [ ] Flink
- [ ] Kernel
- [X] Other (Delta Sharing)

## Description
Upgrade delta-sharing-client to 1.0.5

## How was this patch tested?
Unit Tests

## Does this PR introduce _any_ user-facing changes?
No
  • Loading branch information
linzhou-db committed Apr 24, 2024
1 parent 3c09d95 commit 8b4b6cc
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 2 deletions.
2 changes: 1 addition & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ lazy val sharing = (project in file("sharing"))
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-sql" % defaultSparkVersion % "provided",

"io.delta" %% "delta-sharing-client" % "1.0.4",
"io.delta" %% "delta-sharing-client" % "1.0.5",

// Test deps
"org.scalatest" %% "scalatest" % scalaTestVersion % "test",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import org.apache.spark.sql.delta.sources.{
DeltaSourceOffset
}
import io.delta.sharing.client.DeltaSharingClient
import io.delta.sharing.client.util.ConfUtils
import io.delta.sharing.client.model.{Table => DeltaSharingTable}

import org.apache.spark.delta.sharing.CachedTableManager
Expand Down Expand Up @@ -155,7 +156,17 @@ case class DeltaFormatSharingSource(
private var lastTimestampForGetVersionFromServer: Long = -1

// The minimum gap between two getTableVersion rpcs, to avoid a high traffic load to the server.
private val QUERY_TABLE_VERSION_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(30)
private val QUERY_TABLE_VERSION_INTERVAL_MILLIS = {
val intervalSeconds = ConfUtils.MINIMUM_TABLE_VERSION_INTERVAL_SECONDS.max(
ConfUtils.streamingQueryTableVersionIntervalSeconds(spark.sessionState.conf)
)
logInfo(s"Configured queryTableVersionIntervalSeconds:${intervalSeconds}.")
if (intervalSeconds < ConfUtils.MINIMUM_TABLE_VERSION_INTERVAL_SECONDS) {
throw new IllegalArgumentException(s"QUERY_TABLE_VERSION_INTERVAL_MILLIS($intervalSeconds) " +
s"must not be less than ${ConfUtils.MINIMUM_TABLE_VERSION_INTERVAL_SECONDS} seconds.")
}
TimeUnit.SECONDS.toMillis(intervalSeconds)
}

// Maximum number of versions of getFiles() rpc when fetching files from the server. Used to
// reduce the number of files returned to avoid timeout of the rpc on the server.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,23 @@ class DeltaFormatSharingSourceSuite
.load(tablePath)
.filter($"value" contains "keep")

spark.sessionState.conf.setConfString(
"spark.delta.sharing.streaming.queryTableVersionIntervalSeconds",
"9s"
)
val e = intercept[Exception] {
testStream(df)(
AssertOnQuery { q =>
q.processAllAvailable(); true
}
)
}
assert(e.getMessage.contains("must not be less than 10 seconds"))

spark.sessionState.conf.setConfString(
"spark.delta.sharing.streaming.queryTableVersionIntervalSeconds",
"10s"
)
testStream(df)(
AssertOnQuery { q =>
q.processAllAvailable(); true
Expand Down

0 comments on commit 8b4b6cc

Please sign in to comment.