Skip to content
Permalink
Browse files
feat: add connectionId support for external data sources (#776)
Add connecitonId to ExternalTableDefinition to allow external data source query through the BigQuery client.
  • Loading branch information
stephaniewang526 committed Oct 1, 2020
1 parent 7ce695f commit dcd7daa15d8f820affc9c155ba6d7a9a4c15066d
@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- see http://www.mojohaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
<differences>
<!-- TODO: REMOVE AFTER RELEASE OF CONNECTION_ID FEAT -->
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/ExternalTableDefinition</className>
<method>java.lang.String getConnectionId()</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/ExternalTableDefinition$Builder</className>
<method>com.google.cloud.bigquery.ExternalTableDefinition$Builder setConnectionId(java.lang.String)</method>
</difference>
</differences>
@@ -124,6 +124,12 @@ public Builder setFormatOptions(FormatOptions formatOptions) {
*/
public abstract Builder setCompression(String compression);

/**
* [Optional, Trusted Tester] connectionId for external data source. The value may be {@code
* null}.
*/
public abstract Builder setConnectionId(String connectionId);

/**
* [Experimental] Sets detection of schema and format options automatically. Any option
* specified explicitly will be honored.
@@ -158,6 +164,16 @@ abstract Builder setHivePartitioningOptionsInner(
@Nullable
public abstract String getCompression();

/**
* Returns the connection ID used to connect to external data source.
*
* @see <a
* href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration">
* ConnectionId</a>
*/
@Nullable
public abstract String getConnectionId();

/**
* Returns whether BigQuery should allow extra values that are not represented in the table
* schema. If true, the extra values are ignored. If false, records with extra columns are treated
@@ -249,6 +265,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC
if (getCompression() != null) {
externalConfigurationPb.setCompression(getCompression());
}
if (getConnectionId() != null) {
externalConfigurationPb.setConnectionId(getConnectionId());
}
if (ignoreUnknownValues() != null) {
externalConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues());
}
@@ -415,6 +434,9 @@ static ExternalTableDefinition fromPb(Table tablePb) {
builder.setFormatOptions(FormatOptions.of(externalDataConfiguration.getSourceFormat()));
}
builder.setCompression(externalDataConfiguration.getCompression());
if (externalDataConfiguration.getConnectionId() != null) {
builder.setConnectionId(externalDataConfiguration.getConnectionId());
}
builder.setIgnoreUnknownValues(externalDataConfiguration.getIgnoreUnknownValues());
if (externalDataConfiguration.getCsvOptions() != null) {
builder.setFormatOptions(CsvOptions.fromPb(externalDataConfiguration.getCsvOptions()));
@@ -452,6 +474,9 @@ static ExternalTableDefinition fromExternalDataConfiguration(
if (externalDataConfiguration.getCompression() != null) {
builder.setCompression(externalDataConfiguration.getCompression());
}
if (externalDataConfiguration.getConnectionId() != null) {
builder.setConnectionId(externalDataConfiguration.getConnectionId());
}
if (externalDataConfiguration.getIgnoreUnknownValues() != null) {
builder.setIgnoreUnknownValues(externalDataConfiguration.getIgnoreUnknownValues());
}
@@ -46,6 +46,7 @@ public class ExternalTableDefinitionTest {
private static final Integer MAX_BAD_RECORDS = 42;
private static final Boolean IGNORE_UNKNOWN_VALUES = true;
private static final String COMPRESSION = "GZIP";
private static final String CONNECTION_ID = "123456789";
private static final Boolean AUTODETECT = true;
private static final CsvOptions CSV_OPTIONS = CsvOptions.newBuilder().build();
private static final HivePartitioningOptions HIVE_PARTITIONING_OPTIONS =
@@ -56,6 +57,7 @@ public class ExternalTableDefinitionTest {
private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION =
ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS)
.setCompression(COMPRESSION)
.setConnectionId(CONNECTION_ID)
.setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES)
.setMaxBadRecords(MAX_BAD_RECORDS)
.setAutodetect(AUTODETECT)
@@ -67,10 +69,19 @@ public void testToBuilder() {
compareExternalTableDefinition(
EXTERNAL_TABLE_DEFINITION, EXTERNAL_TABLE_DEFINITION.toBuilder().build());
ExternalTableDefinition externalTableDefinition =
EXTERNAL_TABLE_DEFINITION.toBuilder().setCompression("NONE").build();
EXTERNAL_TABLE_DEFINITION
.toBuilder()
.setCompression("NONE")
.setConnectionId("00000")
.build();
assertEquals("NONE", externalTableDefinition.getCompression());
assertEquals("00000", externalTableDefinition.getConnectionId());
externalTableDefinition =
externalTableDefinition.toBuilder().setCompression(COMPRESSION).build();
externalTableDefinition
.toBuilder()
.setCompression(COMPRESSION)
.setConnectionId(CONNECTION_ID)
.build();
compareExternalTableDefinition(EXTERNAL_TABLE_DEFINITION, externalTableDefinition);
}

@@ -94,6 +105,7 @@ public void testTypeNullPointerException() {
public void testBuilder() {
assertEquals(TableDefinition.Type.EXTERNAL, EXTERNAL_TABLE_DEFINITION.getType());
assertEquals(COMPRESSION, EXTERNAL_TABLE_DEFINITION.getCompression());
assertEquals(CONNECTION_ID, EXTERNAL_TABLE_DEFINITION.getConnectionId());
assertEquals(CSV_OPTIONS, EXTERNAL_TABLE_DEFINITION.getFormatOptions());
assertEquals(IGNORE_UNKNOWN_VALUES, EXTERNAL_TABLE_DEFINITION.ignoreUnknownValues());
assertEquals(MAX_BAD_RECORDS, EXTERNAL_TABLE_DEFINITION.getMaxBadRecords());
@@ -119,6 +131,7 @@ private void compareExternalTableDefinition(
ExternalTableDefinition expected, ExternalTableDefinition value) {
assertEquals(expected, value);
assertEquals(expected.getCompression(), value.getCompression());
assertEquals(expected.getConnectionId(), value.getConnectionId());
assertEquals(expected.getFormatOptions(), value.getFormatOptions());
assertEquals(expected.ignoreUnknownValues(), value.ignoreUnknownValues());
assertEquals(expected.getMaxBadRecords(), value.getMaxBadRecords());

0 comments on commit dcd7daa

Please sign in to comment.