[bigquery] Always use Storage Write API

artie-labs · Jun 14, 2024 · b2524fe · b2524fe
1 parent 355beb5
commit b2524fe
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 7 deletions.
diff --git a/clients/bigquery/bigquery.go b/clients/bigquery/bigquery.go
@@ -34,6 +34,7 @@ const (
 	describeNameCol               = "column_name"
 	describeTypeCol               = "data_type"
 	describeCommentCol            = "description"
+	useStorageWriteAPI            = true
 )
 
 type Store struct {
@@ -71,7 +72,7 @@ func (s *Store) PrepareTemporaryTable(tableData *optimization.TableData, tableCo
 	}
 
 	// Load the data
-	if s.config.BigQuery.UseStorageWriteAPI {
+	if useStorageWriteAPI {
 		return s.putTableViaStorageWriteAPI(context.Background(), bqTempTableID, tableData)
 	} else {
 		return s.putTableViaLegacyAPI(context.Background(), bqTempTableID, tableData)

diff --git a/lib/config/bigquery.go b/lib/config/bigquery.go
@@ -5,12 +5,11 @@ import "fmt"
 type BigQuery struct {
 	// PathToCredentials is _optional_ if you have GOOGLE_APPLICATION_CREDENTIALS set as an env var
 	// Links to credentials: https://cloud.google.com/docs/authentication/application-default-credentials#GAC
-	PathToCredentials  string `yaml:"pathToCredentials"`
-	DefaultDataset     string `yaml:"defaultDataset"`
-	ProjectID          string `yaml:"projectID"`
-	Location           string `yaml:"location"`
-	BatchSize          int    `yaml:"batchSize"`
-	UseStorageWriteAPI bool   `yaml:"__useStorageWriteAPI"` // Not officially supported yet.
+	PathToCredentials string `yaml:"pathToCredentials"`
+	DefaultDataset    string `yaml:"defaultDataset"`
+	ProjectID         string `yaml:"projectID"`
+	Location          string `yaml:"location"`
+	BatchSize         int    `yaml:"batchSize"`
 }
 
 func (b *BigQuery) LoadDefaultValues() {