Skip to content

Commit

Permalink
Add support for writing V2 Checkpoints
Browse files Browse the repository at this point in the history
Adds capability to write V2 Checkpoints. This PR only adds write support. More Unit Tests to follow in an upcoming PR which will add read support.

Closes #2031

GitOrigin-RevId: c3a889b8f67829ec48f6ff650d4d77b75531288f
  • Loading branch information
dhruvarya-db authored and allisonport-db committed Sep 14, 2023
1 parent 201e89b commit 7442ebf
Show file tree
Hide file tree
Showing 7 changed files with 506 additions and 53 deletions.
Expand Up @@ -16,6 +16,7 @@

package org.apache.spark.sql.delta

import org.apache.spark.sql.delta.actions.Protocol
import org.apache.spark.sql.delta.metering.DeltaLogging
import org.apache.spark.sql.delta.sources.DeltaSQLConf
import org.apache.spark.sql.delta.util.FileNames._
Expand Down Expand Up @@ -59,6 +60,19 @@ trait CheckpointProvider extends UninitializedCheckpointProvider {
def allActionsFileIndexes(): Seq[DeltaLogFileIndex]
}

object CheckpointProvider extends DeltaLogging {

private[delta] def isV2CheckpointEnabled(protocol: Protocol): Boolean =
protocol.isFeatureSupported(V2CheckpointTableFeature)

/**
* Returns whether V2 Checkpoints are enabled or not.
* This means an underlying checkpoint in this table could be a V2Checkpoint with sidecar files.
*/
def isV2CheckpointEnabled(snapshotDescriptor: SnapshotDescriptor): Boolean =
isV2CheckpointEnabled(snapshotDescriptor.protocol)
}

/**
* An implementation of [[CheckpointProvider]] where the information about checkpoint files
* (i.e. Seq[FileStatus]) is already known in advance.
Expand Down

0 comments on commit 7442ebf

Please sign in to comment.