forked from googleapis/google-cloud-go
-
Notifications
You must be signed in to change notification settings - Fork 0
/
load_op.go
112 lines (84 loc) · 3.95 KB
/
load_op.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bigquery
import (
"fmt"
"golang.org/x/net/context"
bq "google.golang.org/api/bigquery/v2"
)
type loadOption interface {
customizeLoad(conf *bq.JobConfigurationLoad, projectID string)
}
// DestinationSchema returns an Option that specifies the schema to use when loading data into a new table.
// A DestinationSchema Option must be supplied when loading data from Google Cloud Storage into a non-existent table.
// Caveat: DestinationSchema is not required if the data being loaded is a datastore backup.
// schema must not be nil.
func DestinationSchema(schema Schema) Option { return destSchema{Schema: schema} }
type destSchema struct {
Schema
}
func (opt destSchema) implementsOption() {}
func (opt destSchema) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) {
conf.Schema = opt.asTableSchema()
}
// MaxBadRecords returns an Option that sets the maximum number of bad records that will be ignored.
// If this maximum is exceeded, the operation will be unsuccessful.
func MaxBadRecords(n int64) Option { return maxBadRecords(n) }
type maxBadRecords int64
func (opt maxBadRecords) implementsOption() {}
func (opt maxBadRecords) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) {
conf.MaxBadRecords = int64(opt)
}
// AllowJaggedRows returns an Option that causes missing trailing optional columns to be tolerated in CSV data. Missing values are treated as nulls.
func AllowJaggedRows() Option { return allowJaggedRows{} }
type allowJaggedRows struct{}
func (opt allowJaggedRows) implementsOption() {}
func (opt allowJaggedRows) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) {
conf.AllowJaggedRows = true
}
// AllowQuotedNewlines returns an Option that allows quoted data sections containing newlines in CSV data.
func AllowQuotedNewlines() Option { return allowQuotedNewlines{} }
type allowQuotedNewlines struct{}
func (opt allowQuotedNewlines) implementsOption() {}
func (opt allowQuotedNewlines) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) {
conf.AllowQuotedNewlines = true
}
// IgnoreUnknownValues returns an Option that causes values not matching the schema to be tolerated.
// Unknown values are ignored. For CSV this ignores extra values at the end of a line.
// For JSON this ignores named values that do not match any column name.
// If this Option is not used, records containing unknown values are treated as bad records.
// The MaxBadRecords Option can be used to customize how bad records are handled.
func IgnoreUnknownValues() Option { return ignoreUnknownValues{} }
type ignoreUnknownValues struct{}
func (opt ignoreUnknownValues) implementsOption() {}
func (opt ignoreUnknownValues) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) {
conf.IgnoreUnknownValues = true
}
func (c *Client) load(ctx context.Context, dst *Table, src *GCSReference, options []Option) (*Job, error) {
job, options := initJobProto(c.projectID, options)
payload := &bq.JobConfigurationLoad{}
dst.customizeLoadDst(payload, c.projectID)
src.customizeLoadSrc(payload, c.projectID)
for _, opt := range options {
o, ok := opt.(loadOption)
if !ok {
return nil, fmt.Errorf("option (%#v) not applicable to dst/src pair: dst: %T ; src: %T", opt, dst, src)
}
o.customizeLoad(payload, c.projectID)
}
job.Configuration = &bq.JobConfiguration{
Load: payload,
}
return c.service.insertJob(ctx, job, c.projectID)
}