Skip to content
Permalink
Browse files
feat: add requirePartitionFilter field in HivePartitioningOptions (#803)
  • Loading branch information
Praful Makani committed Oct 15, 2020
1 parent 81aa9a0 commit 193c8860d3a21e56b6ebe77240bd4eee3d19ee72
@@ -23,17 +23,20 @@
public final class HivePartitioningOptions {

private final String mode;
private final Boolean requirePartitionFilter;
private final String sourceUriPrefix;

public static final class Builder {

private String mode;
private Boolean requirePartitionFilter;
private String sourceUriPrefix;

private Builder() {}

private Builder(HivePartitioningOptions options) {
this.mode = options.mode;
this.requirePartitionFilter = options.requirePartitionFilter;
this.sourceUriPrefix = options.sourceUriPrefix;
}

@@ -50,6 +53,17 @@ public Builder setMode(String mode) {
return this;
}

/**
* [Optional] If set to true, queries over this table require a partition filter that can be
* used for partition elimination to be specified. Note that this field should only be true when
* creating a permanent external table or querying a temporary external table. Hive-partitioned
* loads with requirePartitionFilter explicitly set to true will fail.
*/
public Builder setRequirePartitionFilter(Boolean requirePartitionFilter) {
this.requirePartitionFilter = requirePartitionFilter;
return this;
}

/**
* [Optional] When hive partition detection is requested, a common prefix for all source uris
* should be supplied. The prefix must end immediately before the partition key encoding begins.
@@ -72,6 +86,7 @@ public HivePartitioningOptions build() {

private HivePartitioningOptions(Builder builder) {
this.mode = builder.mode;
this.requirePartitionFilter = builder.requirePartitionFilter;
this.sourceUriPrefix = builder.sourceUriPrefix;
}

@@ -80,6 +95,14 @@ public String getMode() {
return mode;
}

/**
* Returns true if a partition filter (that can be used for partition elimination) is required for
* queries over this table.
*/
public Boolean getRequirePartitionFilter() {
return requirePartitionFilter;
}

/* Returns the sourceUriPrefix of hive partitioning */
public String getSourceUriPrefix() {
return sourceUriPrefix;
@@ -99,6 +122,7 @@ public static Builder newBuilder() {
public String toString() {
return MoreObjects.toStringHelper(this)
.add("mode", mode)
.add("requirePartitionFilter", requirePartitionFilter)
.add("sourceUriPrefix", sourceUriPrefix)
.toString();
}
@@ -109,6 +133,8 @@ public boolean equals(Object obj) {
|| obj != null
&& obj.getClass().equals(HivePartitioningOptions.class)
&& Objects.equals(mode, ((HivePartitioningOptions) obj).getMode())
&& Objects.equals(
requirePartitionFilter, ((HivePartitioningOptions) obj).getRequirePartitionFilter())
&& Objects.equals(
sourceUriPrefix, ((HivePartitioningOptions) obj).getSourceUriPrefix());
}
@@ -122,6 +148,7 @@ com.google.api.services.bigquery.model.HivePartitioningOptions toPb() {
com.google.api.services.bigquery.model.HivePartitioningOptions options =
new com.google.api.services.bigquery.model.HivePartitioningOptions();
options.setMode(mode);
options.setRequirePartitionFilter(requirePartitionFilter);
options.setSourceUriPrefix(sourceUriPrefix);
return options;
}
@@ -132,6 +159,9 @@ static HivePartitioningOptions fromPb(
if (options.getMode() != null) {
builder.setMode(options.getMode());
}
if (options.getRequirePartitionFilter() != null) {
builder.setRequirePartitionFilter(options.getRequirePartitionFilter());
}
if (options.getSourceUriPrefix() != null) {
builder.setSourceUriPrefix(options.getSourceUriPrefix());
}
@@ -24,9 +24,11 @@ public class HivePartitioningOptionsTest {

private static final String MODE = "STRING";
private static final String SOURCE_URI_PREFIX = "gs://bucket/path_to_table";
private static final Boolean REQUIRE_PARTITION_FILTER = true;
private static final HivePartitioningOptions HIVE_PARTITIONING_OPTIONS =
HivePartitioningOptions.newBuilder()
.setMode(MODE)
.setRequirePartitionFilter(REQUIRE_PARTITION_FILTER)
.setSourceUriPrefix(SOURCE_URI_PREFIX)
.build();

@@ -49,6 +51,8 @@ public void testToBuilderIncomplete() {
@Test
public void testBuilder() {
assertThat(HIVE_PARTITIONING_OPTIONS.getMode()).isEqualTo(MODE);
assertThat(HIVE_PARTITIONING_OPTIONS.getRequirePartitionFilter())
.isEqualTo(REQUIRE_PARTITION_FILTER);
assertThat(HIVE_PARTITIONING_OPTIONS.getSourceUriPrefix()).isEqualTo(SOURCE_URI_PREFIX);
}

@@ -62,6 +66,7 @@ public void testToAndFromPb() {
private void compareHivePartitioningOptions(
HivePartitioningOptions expected, HivePartitioningOptions value) {
assertThat(value.getMode()).isEqualTo(expected.getMode());
assertThat(value.getRequirePartitionFilter()).isEqualTo(expected.getRequirePartitionFilter());
assertThat(value.getSourceUriPrefix()).isEqualTo(expected.getSourceUriPrefix());
assertThat(value.toString()).isEqualTo(expected.toString());
assertThat(value.hashCode()).isEqualTo(expected.hashCode());
@@ -63,7 +63,7 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<github.global.server>github</github.global.server>
<site.installationModule>google-cloud-bigquery-parent</site.installationModule>
<google-api-services-bigquery.version>v2-rev20200916-1.30.10
<google-api-services-bigquery.version>v2-rev20200925-1.30.10
</google-api-services-bigquery.version>
</properties>

0 comments on commit 193c886

Please sign in to comment.