Skip to content

Commit

Permalink
bigquery: export HiveParititioningOptions in load job configurations
Browse files Browse the repository at this point in the history
  • Loading branch information
shollyman committed Mar 31, 2021
1 parent 7aeee23 commit 2b01606
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 0 deletions.
10 changes: 10 additions & 0 deletions bigquery/load.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ type LoadConfig struct {
// For ingestion from datastore backups, ProjectionFields governs which fields
// are projected from the backup. The default behavior projects all fields.
ProjectionFields []string

// HivePartitioningOptions allows use of Hive partitioning based on the
// layout of objects in Google Cloud Storage.
HivePartitioningOptions *HivePartitioningOptions
}

func (l *LoadConfig) toBQ() (*bq.JobConfiguration, io.Reader) {
Expand All @@ -83,6 +87,9 @@ func (l *LoadConfig) toBQ() (*bq.JobConfiguration, io.Reader) {
ProjectionFields: l.ProjectionFields,
},
}
if l.HivePartitioningOptions != nil {
config.Load.HivePartitioningOptions = l.HivePartitioningOptions.toBQ()
}
media := l.Src.populateLoadConfig(config.Load)
return config, media
}
Expand Down Expand Up @@ -111,6 +118,9 @@ func bqToLoadConfig(q *bq.JobConfiguration, c *Client) *LoadConfig {
fc = &s.FileConfig
lc.Src = s
}
if q.Load.HivePartitioningOptions != nil {
lc.HivePartitioningOptions = bqToHivePartitioningOptions(q.Load.HivePartitioningOptions)
}
bqPopulateFileConfig(q.Load, fc)
return lc
}
Expand Down
25 changes: 25 additions & 0 deletions bigquery/load_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,31 @@ func TestLoad(t *testing.T) {
return j
}(),
},
{
dst: c.Dataset("dataset-id").Table("table-id"),
src: func() *GCSReference {
g := NewGCSReference("uri")
g.SourceFormat = Parquet
return g
}(),
config: LoadConfig{
HivePartitioningOptions: &HivePartitioningOptions{
Mode: CustomHivePartitioningMode,
SourceURIPrefix: "source_uri",
RequirePartitionFilter: true,
},
},
want: func() *bq.Job {
j := defaultLoadJob()
j.Configuration.Load.SourceFormat = "PARQUET"
j.Configuration.Load.HivePartitioningOptions = &bq.HivePartitioningOptions{
Mode: "CUSTOM",
RequirePartitionFilter: true,
SourceUriPrefix: "source_uri",
}
return j
}(),
},
}

for i, tc := range testCases {
Expand Down

0 comments on commit 2b01606

Please sign in to comment.