Skip to content

Commit ea3cde5

Browse files
shollymanshollyman
authored
feat(bigquery): add support for bignumeric (#2779)
feat(bigquery): add support for bignumeric This PR adds basic support for the BIGNUMERIC type in BigQuery. This library has several cases where it tries to infer the appropriate BigQuery type from a native Go type. For big.Rat types, we continue the existing behavior of mapping them to NUMERIC, as the big.Rat doesn't provide a general way of indicating desired precision or scale to determine whether BIGNUMERIC is a more appropriate mapping.
1 parent a781a3a commit ea3cde5

File tree

7 files changed

+172
-95
lines changed

7 files changed

+172
-95
lines changed

Diff for: ‎bigquery/integration_test.go

+15-13
Original file line numberDiff line numberDiff line change
@@ -1434,6 +1434,7 @@ func TestIntegration_InsertAndReadNullable(t *testing.T) {
14341434
ctm := civil.Time{Hour: 15, Minute: 4, Second: 5, Nanosecond: 6000}
14351435
cdt := civil.DateTime{Date: testDate, Time: ctm}
14361436
rat := big.NewRat(33, 100)
1437+
rat2 := big.NewRat(66, 100)
14371438
geo := "POINT(-122.198939 47.669865)"
14381439

14391440
// Nil fields in the struct.
@@ -1455,20 +1456,21 @@ func TestIntegration_InsertAndReadNullable(t *testing.T) {
14551456

14561457
// Populate the struct with values.
14571458
testInsertAndReadNullable(t, testStructNullable{
1458-
String: NullString{"x", true},
1459-
Bytes: []byte{1, 2, 3},
1460-
Integer: NullInt64{1, true},
1461-
Float: NullFloat64{2.3, true},
1462-
Boolean: NullBool{true, true},
1463-
Timestamp: NullTimestamp{testTimestamp, true},
1464-
Date: NullDate{testDate, true},
1465-
Time: NullTime{ctm, true},
1466-
DateTime: NullDateTime{cdt, true},
1467-
Numeric: rat,
1468-
Geography: NullGeography{geo, true},
1469-
Record: &subNullable{X: NullInt64{4, true}},
1459+
String: NullString{"x", true},
1460+
Bytes: []byte{1, 2, 3},
1461+
Integer: NullInt64{1, true},
1462+
Float: NullFloat64{2.3, true},
1463+
Boolean: NullBool{true, true},
1464+
Timestamp: NullTimestamp{testTimestamp, true},
1465+
Date: NullDate{testDate, true},
1466+
Time: NullTime{ctm, true},
1467+
DateTime: NullDateTime{cdt, true},
1468+
Numeric: rat,
1469+
BigNumeric: rat2,
1470+
Geography: NullGeography{geo, true},
1471+
Record: &subNullable{X: NullInt64{4, true}},
14701472
},
1471-
[]Value{"x", []byte{1, 2, 3}, int64(1), 2.3, true, testTimestamp, testDate, ctm, cdt, rat, geo, []Value{int64(4)}})
1473+
[]Value{"x", []byte{1, 2, 3}, int64(1), 2.3, true, testTimestamp, testDate, ctm, cdt, rat, rat2, geo, []Value{int64(4)}})
14721474
}
14731475

14741476
func testInsertAndReadNullable(t *testing.T, ts testStructNullable, wantRow []Value) {

Diff for: ‎bigquery/params.go

+23-18
Original file line numberDiff line numberDiff line change
@@ -65,16 +65,17 @@ func (e invalidFieldNameError) Error() string {
6565
var fieldCache = fields.NewCache(bqTagParser, nil, nil)
6666

6767
var (
68-
int64ParamType = &bq.QueryParameterType{Type: "INT64"}
69-
float64ParamType = &bq.QueryParameterType{Type: "FLOAT64"}
70-
boolParamType = &bq.QueryParameterType{Type: "BOOL"}
71-
stringParamType = &bq.QueryParameterType{Type: "STRING"}
72-
bytesParamType = &bq.QueryParameterType{Type: "BYTES"}
73-
dateParamType = &bq.QueryParameterType{Type: "DATE"}
74-
timeParamType = &bq.QueryParameterType{Type: "TIME"}
75-
dateTimeParamType = &bq.QueryParameterType{Type: "DATETIME"}
76-
timestampParamType = &bq.QueryParameterType{Type: "TIMESTAMP"}
77-
numericParamType = &bq.QueryParameterType{Type: "NUMERIC"}
68+
int64ParamType = &bq.QueryParameterType{Type: "INT64"}
69+
float64ParamType = &bq.QueryParameterType{Type: "FLOAT64"}
70+
boolParamType = &bq.QueryParameterType{Type: "BOOL"}
71+
stringParamType = &bq.QueryParameterType{Type: "STRING"}
72+
bytesParamType = &bq.QueryParameterType{Type: "BYTES"}
73+
dateParamType = &bq.QueryParameterType{Type: "DATE"}
74+
timeParamType = &bq.QueryParameterType{Type: "TIME"}
75+
dateTimeParamType = &bq.QueryParameterType{Type: "DATETIME"}
76+
timestampParamType = &bq.QueryParameterType{Type: "TIMESTAMP"}
77+
numericParamType = &bq.QueryParameterType{Type: "NUMERIC"}
78+
bigNumericParamType = &bq.QueryParameterType{Type: "BIGNUMERIC"}
7879
)
7980

8081
var (
@@ -233,6 +234,9 @@ func paramValue(v reflect.Value) (bq.QueryParameterValue, error) {
233234
return res, nil
234235

235236
case typeOfRat:
237+
// big.Rat types don't communicate scale or precision, so we cannot
238+
// disambiguate between NUMERIC and BIGNUMERIC. For now, we'll continue
239+
// to honor previous behavior and send as Numeric type.
236240
res.Value = NumericString(v.Interface().(*big.Rat))
237241
return res, nil
238242
}
@@ -304,14 +308,15 @@ func bqToQueryParameter(q *bq.QueryParameter) (QueryParameter, error) {
304308
}
305309

306310
var paramTypeToFieldType = map[string]FieldType{
307-
int64ParamType.Type: IntegerFieldType,
308-
float64ParamType.Type: FloatFieldType,
309-
boolParamType.Type: BooleanFieldType,
310-
stringParamType.Type: StringFieldType,
311-
bytesParamType.Type: BytesFieldType,
312-
dateParamType.Type: DateFieldType,
313-
timeParamType.Type: TimeFieldType,
314-
numericParamType.Type: NumericFieldType,
311+
int64ParamType.Type: IntegerFieldType,
312+
float64ParamType.Type: FloatFieldType,
313+
boolParamType.Type: BooleanFieldType,
314+
stringParamType.Type: StringFieldType,
315+
bytesParamType.Type: BytesFieldType,
316+
dateParamType.Type: DateFieldType,
317+
timeParamType.Type: TimeFieldType,
318+
numericParamType.Type: NumericFieldType,
319+
bigNumericParamType.Type: BigNumericFieldType,
315320
}
316321

317322
// Convert a parameter value from the service to a Go value. This is similar to, but

Diff for: ‎bigquery/schema.go

+20-12
Original file line numberDiff line numberDiff line change
@@ -182,23 +182,27 @@ const (
182182
// GeographyFieldType is a string field type. Geography types represent a set of points
183183
// on the Earth's surface, represented in Well Known Text (WKT) format.
184184
GeographyFieldType FieldType = "GEOGRAPHY"
185+
// BigNumericFieldType is a numeric field type that supports values of larger precision
186+
// and scale than the NumericFieldType.
187+
BigNumericFieldType FieldType = "BIGNUMERIC"
185188
)
186189

187190
var (
188191
errEmptyJSONSchema = errors.New("bigquery: empty JSON schema")
189192
fieldTypes = map[FieldType]bool{
190-
StringFieldType: true,
191-
BytesFieldType: true,
192-
IntegerFieldType: true,
193-
FloatFieldType: true,
194-
BooleanFieldType: true,
195-
TimestampFieldType: true,
196-
RecordFieldType: true,
197-
DateFieldType: true,
198-
TimeFieldType: true,
199-
DateTimeFieldType: true,
200-
NumericFieldType: true,
201-
GeographyFieldType: true,
193+
StringFieldType: true,
194+
BytesFieldType: true,
195+
IntegerFieldType: true,
196+
FloatFieldType: true,
197+
BooleanFieldType: true,
198+
TimestampFieldType: true,
199+
RecordFieldType: true,
200+
DateFieldType: true,
201+
TimeFieldType: true,
202+
DateTimeFieldType: true,
203+
NumericFieldType: true,
204+
GeographyFieldType: true,
205+
BigNumericFieldType: true,
202206
}
203207
// The API will accept alias names for the types based on the Standard SQL type names.
204208
fieldAliases = map[FieldType]FieldType{
@@ -346,6 +350,10 @@ func inferFieldSchema(fieldName string, rt reflect.Type, nullable bool) (*FieldS
346350
case typeOfDateTime:
347351
return &FieldSchema{Required: true, Type: DateTimeFieldType}, nil
348352
case typeOfRat:
353+
// We automatically infer big.Rat values as NUMERIC as we cannot
354+
// determine precision/scale from the type. Users who want the
355+
// larger precision of BIGNUMERIC need to manipulate the inferred
356+
// schema.
349357
return &FieldSchema{Required: !nullable, Type: NumericFieldType}, nil
350358
}
351359
if ft := nullableFieldType(rt); ft != "" {

Diff for: ‎bigquery/schema_test.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -1041,7 +1041,8 @@ func TestSchemaFromJSON(t *testing.T) {
10411041
{"name":"flat_date","type":"DATE","mode":"NULLABLE","description":"Flat required DATE"},
10421042
{"name":"flat_time","type":"TIME","mode":"REQUIRED","description":"Flat nullable TIME"},
10431043
{"name":"flat_datetime","type":"DATETIME","mode":"NULLABLE","description":"Flat required DATETIME"},
1044-
{"name":"flat_numeric","type":"NUMERIC","mode":"REQUIRED","description":"Flat nullable NUMERIC"},
1044+
{"name":"flat_numeric","type":"NUMERIC","mode":"REQUIRED","description":"Flat required NUMERIC"},
1045+
{"name":"flat_bignumeric","type":"BIGNUMERIC","mode":"NULLABLE","description":"Flat nullable BIGNUMERIC"},
10451046
{"name":"flat_geography","type":"GEOGRAPHY","mode":"REQUIRED","description":"Flat required GEOGRAPHY"},
10461047
{"name":"aliased_integer","type":"INT64","mode":"REQUIRED","description":"Aliased required integer"},
10471048
{"name":"aliased_boolean","type":"BOOL","mode":"NULLABLE","description":"Aliased nullable boolean"},
@@ -1058,7 +1059,8 @@ func TestSchemaFromJSON(t *testing.T) {
10581059
fieldSchema("Flat required DATE", "flat_date", "DATE", false, false, nil),
10591060
fieldSchema("Flat nullable TIME", "flat_time", "TIME", false, true, nil),
10601061
fieldSchema("Flat required DATETIME", "flat_datetime", "DATETIME", false, false, nil),
1061-
fieldSchema("Flat nullable NUMERIC", "flat_numeric", "NUMERIC", false, true, nil),
1062+
fieldSchema("Flat required NUMERIC", "flat_numeric", "NUMERIC", false, true, nil),
1063+
fieldSchema("Flat nullable BIGNUMERIC", "flat_bignumeric", "BIGNUMERIC", false, false, nil),
10621064
fieldSchema("Flat required GEOGRAPHY", "flat_geography", "GEOGRAPHY", false, true, nil),
10631065
fieldSchema("Aliased required integer", "aliased_integer", "INTEGER", false, true, nil),
10641066
fieldSchema("Aliased nullable boolean", "aliased_boolean", "BOOLEAN", false, false, nil),

Diff for: ‎bigquery/value.go

+33-1
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,13 @@ func determineSetFunc(ftype reflect.Type, stype FieldType) setFunc {
407407
return setNull(v, x, func() interface{} { return x.(*big.Rat) })
408408
}
409409
}
410+
411+
case BigNumericFieldType:
412+
if ftype == typeOfRat {
413+
return func(v reflect.Value, x interface{}) error {
414+
return setNull(v, x, func() interface{} { return x.(*big.Rat) })
415+
}
416+
}
410417
}
411418
return nil
412419
}
@@ -692,7 +699,7 @@ func structFieldToUploadValue(vfield reflect.Value, schemaField *FieldSchema) (i
692699
}
693700

694701
func toUploadValue(val interface{}, fs *FieldSchema) interface{} {
695-
if fs.Type == TimeFieldType || fs.Type == DateTimeFieldType || fs.Type == NumericFieldType {
702+
if fs.Type == TimeFieldType || fs.Type == DateTimeFieldType || fs.Type == NumericFieldType || fs.Type == BigNumericFieldType {
696703
return toUploadValueReflect(reflect.ValueOf(val), fs)
697704
}
698705
return val
@@ -721,6 +728,13 @@ func toUploadValueReflect(v reflect.Value, fs *FieldSchema) interface{} {
721728
return formatUploadValue(v, fs, func(v reflect.Value) string {
722729
return NumericString(v.Interface().(*big.Rat))
723730
})
731+
case BigNumericFieldType:
732+
if r, ok := v.Interface().(*big.Rat); ok && r == nil {
733+
return nil
734+
}
735+
return formatUploadValue(v, fs, func(v reflect.Value) string {
736+
return BigNumericString(v.Interface().(*big.Rat))
737+
})
724738
default:
725739
if !fs.Repeated || v.Len() > 0 {
726740
return v.Interface()
@@ -786,6 +800,12 @@ const (
786800

787801
// NumericScaleDigits is the maximum number of digits after the decimal point in a NUMERIC value.
788802
NumericScaleDigits = 9
803+
804+
// BigNumericPrecisionDigits is the maximum number of full digits in a BIGNUMERIC value.
805+
BigNumericPrecisionDigits = 76
806+
807+
// BigNumericScaleDigits is the maximum number of full digits in a BIGNUMERIC value.
808+
BigNumericScaleDigits = 38
789809
)
790810

791811
// NumericString returns a string representing a *big.Rat in a format compatible
@@ -795,6 +815,12 @@ func NumericString(r *big.Rat) string {
795815
return r.FloatString(NumericScaleDigits)
796816
}
797817

818+
// BigNumericString returns a string representing a *big.Rat in a format compatible with BigQuery
819+
// SQL. It returns a floating point literal with 38 digits after the decimal point.
820+
func BigNumericString(r *big.Rat) string {
821+
return r.FloatString(BigNumericScaleDigits)
822+
}
823+
798824
// convertRows converts a series of TableRows into a series of Value slices.
799825
// schema is used to interpret the data from rows; its length must match the
800826
// length of each row.
@@ -913,6 +939,12 @@ func convertBasicType(val string, typ FieldType) (Value, error) {
913939
return nil, fmt.Errorf("bigquery: invalid NUMERIC value %q", val)
914940
}
915941
return Value(r), nil
942+
case BigNumericFieldType:
943+
r, ok := (&big.Rat{}).SetString(val)
944+
if !ok {
945+
return nil, fmt.Errorf("bigquery: invalid BIGNUMERIC value %q", val)
946+
}
947+
return Value(r), nil
916948
case GeographyFieldType:
917949
return val, nil
918950
default:

0 commit comments

Comments
 (0)