Skip to content
Permalink
Browse files
feat: add support for Hive partitioning options when creating externa…
…l tables (#235) (#236)
  • Loading branch information
ekanth committed Mar 20, 2020
1 parent a1e0656 commit ccc2bb3de28a36e3791d5441c8bdea2333877ee8
@@ -135,6 +135,14 @@ public Builder setFormatOptions(FormatOptions formatOptions) {
/** Sets the table schema. */
public abstract Builder setSchema(Schema schema);

/** Sets the table Hive partitioning options. */
public Builder setHivePartitioningOptions(HivePartitioningOptions hivePartitioningOptions) {
return setHivePartitioningOptionsInner(hivePartitioningOptions);
};

abstract Builder setHivePartitioningOptionsInner(
HivePartitioningOptions hivePartitioningOptions);

/** Creates an {@code ExternalTableDefinition} object. */
@Override
public abstract ExternalTableDefinition build();
@@ -212,6 +220,19 @@ public <F extends FormatOptions> F getFormatOptions() {
@Nullable
public abstract Boolean getAutodetect();

/**
* [Experimental] Returns the HivePartitioningOptions when the data layout follows Hive
* partitioning convention
*/
@SuppressWarnings("unchecked")
@Nullable
public HivePartitioningOptions getHivePartitioningOptions() {
return getHivePartitioningOptionsInner();
}

@Nullable
abstract HivePartitioningOptions getHivePartitioningOptionsInner();

/** Returns a builder for the {@code ExternalTableDefinition} object. */
public abstract Builder toBuilder();

@@ -257,6 +278,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC
if (getAutodetect() != null) {
externalConfigurationPb.setAutodetect(getAutodetect());
}
if (getHivePartitioningOptions() != null) {
externalConfigurationPb.setHivePartitioningOptions(getHivePartitioningOptions().toPb());
}
return externalConfigurationPb;
}

@@ -405,6 +429,10 @@ static ExternalTableDefinition fromPb(Table tablePb) {
}
builder.setMaxBadRecords(externalDataConfiguration.getMaxBadRecords());
builder.setAutodetect(externalDataConfiguration.getAutodetect());
if (externalDataConfiguration.getHivePartitioningOptions() != null) {
builder.setHivePartitioningOptions(
HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions()));
}
}
return builder.build();
}
@@ -444,6 +472,10 @@ static ExternalTableDefinition fromExternalDataConfiguration(
if (externalDataConfiguration.getAutodetect() != null) {
builder.setAutodetect(externalDataConfiguration.getAutodetect());
}
if (externalDataConfiguration.getHivePartitioningOptions() != null) {
builder.setHivePartitioningOptions(
HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions()));
}
return builder.build();
}
}
@@ -103,6 +103,16 @@ public String toString() {
.toString();
}

@Override
public boolean equals(Object obj) {
return obj == this
|| obj != null
&& obj.getClass().equals(HivePartitioningOptions.class)
&& Objects.equals(mode, ((HivePartitioningOptions) obj).getMode())
&& Objects.equals(
sourceUriPrefix, ((HivePartitioningOptions) obj).getSourceUriPrefix());
}

@Override
public int hashCode() {
return Objects.hash(mode, sourceUriPrefix);
@@ -46,12 +46,18 @@ public class ExternalTableDefinitionTest {
private static final String COMPRESSION = "GZIP";
private static final Boolean AUTODETECT = true;
private static final CsvOptions CSV_OPTIONS = CsvOptions.newBuilder().build();
private static final HivePartitioningOptions HIVE_PARTITIONING_OPTIONS =
HivePartitioningOptions.newBuilder()
.setMode("AUTO")
.setSourceUriPrefix(SOURCE_URIS.get(0))
.build();
private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION =
ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS)
.setCompression(COMPRESSION)
.setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES)
.setMaxBadRecords(MAX_BAD_RECORDS)
.setAutodetect(AUTODETECT)
.setHivePartitioningOptions(HIVE_PARTITIONING_OPTIONS)
.build();

@Test
@@ -83,6 +89,7 @@ public void testBuilder() {
assertEquals(TABLE_SCHEMA, EXTERNAL_TABLE_DEFINITION.getSchema());
assertEquals(SOURCE_URIS, EXTERNAL_TABLE_DEFINITION.getSourceUris());
assertEquals(AUTODETECT, EXTERNAL_TABLE_DEFINITION.getAutodetect());
assertEquals(HIVE_PARTITIONING_OPTIONS, EXTERNAL_TABLE_DEFINITION.getHivePartitioningOptions());
}

@Test
@@ -107,5 +114,6 @@ private void compareExternalTableDefinition(
assertEquals(expected.getSourceUris(), value.getSourceUris());
assertEquals(expected.hashCode(), value.hashCode());
assertEquals(expected.getAutodetect(), value.getAutodetect());
assertEquals(expected.getHivePartitioningOptions(), value.getHivePartitioningOptions());
}
}

0 comments on commit ccc2bb3

Please sign in to comment.