Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

[16617] Source BigQuery: added user agent header #16902

Merged
merged 8 commits into from
Sep 22, 2022
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@
- name: BigQuery
sourceDefinitionId: bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c
dockerRepository: airbyte/source-bigquery
dockerImageTag: 0.2.1
dockerImageTag: 0.2.2
documentationUrl: https://docs.airbyte.io/integrations/sources/bigquery
icon: bigquery.svg
sourceType: database
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1268,7 +1268,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-bigquery:0.2.1"
- dockerImage: "airbyte/source-bigquery:0.2.2"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/bigquery"
connectionSpecification:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
package io.airbyte.db.bigquery;

import static java.util.Objects.isNull;
import static org.apache.commons.lang3.StringUtils.EMPTY;

import com.fasterxml.jackson.databind.JsonNode;
import com.google.api.gax.retrying.RetrySettings;
Expand All @@ -20,7 +21,9 @@
import com.google.cloud.bigquery.StandardSQLTypeName;
import com.google.cloud.bigquery.Table;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Streams;
import io.airbyte.config.WorkerEnvConstants;
import io.airbyte.db.SqlDatabase;
import java.io.ByteArrayInputStream;
import java.io.IOException;
Expand All @@ -29,9 +32,11 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -40,6 +45,7 @@
public class BigQueryDatabase extends SqlDatabase {

private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDatabase.class);
private static final String AGENT_TEMPLATE = "%s (GPN: Airbyte; staging)";

private final BigQuery bigQuery;
private final BigQuerySourceOperations sourceOperations;
Expand All @@ -60,6 +66,7 @@ public BigQueryDatabase(final String projectId, final String jsonCreds, final Bi
bigQuery = bigQueryBuilder
.setProjectId(projectId)
.setCredentials(!isNull(credentials) ? credentials : ServiceAccountCredentials.getApplicationDefault())
.setHeaderProvider(() -> ImmutableMap.of("user-agent", getUserAgentHeader(getConnectorVersion())))
.setRetrySettings(RetrySettings
.newBuilder()
.setMaxAttempts(10)
Expand All @@ -73,6 +80,16 @@ public BigQueryDatabase(final String projectId, final String jsonCreds, final Bi
}
}

private String getUserAgentHeader(String connectorVersion) {
return String.format(AGENT_TEMPLATE, connectorVersion);
}

private String getConnectorVersion() {
return Optional.ofNullable(System.getenv(WorkerEnvConstants.WORKER_CONNECTOR_IMAGE))
.orElse(EMPTY)
.replace("airbyte/", EMPTY).replace(":", "/");
}

@Override
public void execute(final String sql) throws SQLException {
final ImmutablePair<Job, String> result = executeQuery(bigQuery, getQueryConfig(sql, Collections.emptyList()));
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-bigquery/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@ ENV APPLICATION source-bigquery
COPY --from=build /airbyte /airbyte

# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile.
LABEL io.airbyte.version=0.2.1
LABEL io.airbyte.version=0.2.2
LABEL io.airbyte.name=airbyte/source-bigquery
1 change: 1 addition & 0 deletions docs/integrations/sources/bigquery.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ Once you've configured BigQuery as a source, delete the Service Account Key from

| Version | Date | Pull Request | Subject |
| :--- | :--- | :--- | :--- |
| 0.2.2 | 2022-09-22 | [16902](https://github.com/airbytehq/airbyte/pull/16902) | Source BigQuery: added user agent header |
| 0.2.1 | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage |
| 0.2.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. |
| 0.1.9 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors |
Expand Down