Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Core: Add MetadataUpdate.applyTo to re-apply changes #4320

Merged
merged 3 commits into from Mar 15, 2022
Merged

Core: Add MetadataUpdate.applyTo to re-apply changes #4320

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
33f69c1
Core: Add MetadataUpdate.applyTo to re-apply changes.
rdblue Mar 13, 2022
0a29b78
Fix hidden field.
rdblue Mar 13, 2022
98e53e1
Address review comments.
rdblue Mar 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
81 changes: 80 additions & 1 deletion core/src/main/java/org/apache/iceberg/MetadataUpdate.java
View file
Open in desktop
Expand Up @@ -22,11 +22,14 @@
import java.io.Serializable;
import java.util.Map;
import java.util.Set;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;

/**
* Represents a change to table metadata.
*/
public interface MetadataUpdate extends Serializable {
void applyTo(TableMetadata.Builder metadataBuilder);

class AssignUUID implements MetadataUpdate {
private final String uuid;

Expand All @@ -37,6 +40,11 @@ public AssignUUID(String uuid) {
public String uuid() {
return uuid;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
throw new UnsupportedOperationException("Not implemented");
Copy link
Contributor Author

@rdblue rdblue Mar 14, 2022 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not part of the REST API, so I left it unimplemented. I'm not sure whether we will need this because tables have had UUIDs for a long time (added in #264). It seems unlikely that someone will update from 0.6.0 or earlier directly to 0.14.0 and want to use the REST catalog.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree that it is unlikely that will be the case. And existing tables without a UUID will have a UUID added on successful snapshot commit, so people would need to go directly from 0.6.0 to 0.14.0 and then try to use the REST api if I understand correctly.

I think it's fine to ignore that very specific edge case, and if somebody has a complaint, a procedure / utility function can be provided that will simply add a UUID as a direct table maintenance operation or something.

But I agree it's very unlikely for this to occur and probably better not to pollute the code to take it into account.

}
}

class UpgradeFormatVersion implements MetadataUpdate {
Expand All @@ -49,6 +57,11 @@ public UpgradeFormatVersion(int formatVersion) {
public int formatVersion() {
return formatVersion;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.upgradeFormatVersion(formatVersion);
}
}

class AddSchema implements MetadataUpdate {
Expand All @@ -67,6 +80,11 @@ public Schema schema() {
public int lastColumnId() {
return lastColumnId;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.addSchema(schema, lastColumnId);
}
}

class SetCurrentSchema implements MetadataUpdate {
Expand All @@ -79,6 +97,11 @@ public SetCurrentSchema(int schemaId) {
public int schemaId() {
return schemaId;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setCurrentSchema(schemaId);
}
}

class AddPartitionSpec implements MetadataUpdate {
Expand All @@ -91,6 +114,11 @@ public AddPartitionSpec(PartitionSpec spec) {
public PartitionSpec spec() {
return spec;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.addPartitionSpec(spec);
}
}

class SetDefaultPartitionSpec implements MetadataUpdate {
Expand All @@ -103,6 +131,11 @@ public SetDefaultPartitionSpec(int schemaId) {
public int specId() {
return specId;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setDefaultPartitionSpec(specId);
}
}

class AddSortOrder implements MetadataUpdate {
Expand All @@ -112,9 +145,14 @@ public AddSortOrder(SortOrder sortOrder) {
this.sortOrder = sortOrder;
}

public SortOrder spec() {
public SortOrder sortOrder() {
return sortOrder;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.addSortOrder(sortOrder);
}
}

class SetDefaultSortOrder implements MetadataUpdate {
Expand All @@ -127,6 +165,11 @@ public SetDefaultSortOrder(int sortOrderId) {
public int sortOrderId() {
return sortOrderId;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setDefaultSortOrder(sortOrderId);
}
}

class AddSnapshot implements MetadataUpdate {
Expand All @@ -139,6 +182,11 @@ public AddSnapshot(Snapshot snapshot) {
public Snapshot snapshot() {
return snapshot;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.addSnapshot(snapshot);
}
}

class RemoveSnapshot implements MetadataUpdate {
Expand All @@ -151,6 +199,11 @@ public RemoveSnapshot(long snapshotId) {
public long snapshotId() {
return snapshotId;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.removeSnapshots(ImmutableSet.of(snapshotId));
}
}

class RemoveSnapshotRef implements MetadataUpdate {
Expand All @@ -163,6 +216,12 @@ public RemoveSnapshotRef(String name) {
public String name() {
return name;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
// TODO: this should be generalized when tagging is supported
metadataBuilder.removeBranch(name);
}
}

class SetSnapshotRef implements MetadataUpdate {
Expand All @@ -181,6 +240,11 @@ public String name() {
public long snapshotId() {
return snapshotId;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setBranchSnapshot(snapshotId, name);
}
}

class SetProperties implements MetadataUpdate {
Expand All @@ -193,6 +257,11 @@ public SetProperties(Map<String, String> updated) {
public Map<String, String> updated() {
return updated;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setProperties(updated);
}
}

class RemoveProperties implements MetadataUpdate {
Expand All @@ -205,6 +274,11 @@ public RemoveProperties(Set<String> removed) {
public Set<String> removed() {
return removed;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.removeProperties(removed);
}
}

class SetLocation implements MetadataUpdate {
Expand All @@ -217,5 +291,10 @@ public SetLocation(String location) {
public String location() {
return location;
}

@Override
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.setLocation(location);
}
}
}