From 94cfe5936de53773a31d8d2ed378a6f35caf3e74 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Tue, 4 Aug 2026 12:30:04 +0100 Subject: [PATCH] fix: Preserve int64 precision when setting nested scalars from JSON --- scalar/float.go | 3 +++ scalar/int.go | 3 +++ scalar/json_number.go | 14 ++++++++++++++ scalar/json_number_test.go | 27 +++++++++++++++++++++++++++ scalar/list.go | 5 ++--- scalar/struct.go | 4 ++-- scalar/uint.go | 3 +++ 7 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 scalar/json_number.go create mode 100644 scalar/json_number_test.go diff --git a/scalar/float.go b/scalar/float.go index fda2856a68..76557ce7db 100644 --- a/scalar/float.go +++ b/scalar/float.go @@ -1,6 +1,7 @@ package scalar import ( + "encoding/json" "math" "strconv" @@ -119,6 +120,8 @@ func (s *Float) Set(val any) error { return err } s.Value = value + case json.Number: + return s.Set(string(value)) case string: v, err := strconv.ParseFloat(value, 64) if err != nil { diff --git a/scalar/int.go b/scalar/int.go index 4d0f22bf45..1d399d9eb5 100644 --- a/scalar/int.go +++ b/scalar/int.go @@ -1,6 +1,7 @@ package scalar import ( + "encoding/json" "math" "strconv" @@ -124,6 +125,8 @@ func (s *Int) Set(val any) error { return s.Set(int64(value)) case float64: return s.Set(int64(value)) + case json.Number: + return s.Set(string(value)) case string: v, err := strconv.ParseInt(value, 10, 64) if err != nil { diff --git a/scalar/json_number.go b/scalar/json_number.go new file mode 100644 index 0000000000..12a3c97824 --- /dev/null +++ b/scalar/json_number.go @@ -0,0 +1,14 @@ +package scalar + +import ( + "bytes" + "encoding/json" +) + +// unmarshalJSONWithNumbers decodes JSON with UseNumber so int64/uint64 values that +// exceed float64 precision survive as json.Number instead of being rounded. +func unmarshalJSONWithNumbers(data []byte, v any) error { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + return dec.Decode(v) +} diff --git a/scalar/json_number_test.go b/scalar/json_number_test.go new file mode 100644 index 0000000000..2543da5672 --- /dev/null +++ b/scalar/json_number_test.go @@ -0,0 +1,27 @@ +package scalar + +import ( + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/stretchr/testify/require" +) + +func TestNestedIntPrecision(t *testing.T) { + for _, tc := range []struct { + name string + dt arrow.DataType + str string + }{ + {name: "struct_int64", dt: arrow.StructOf(arrow.Field{Name: "v", Type: arrow.PrimitiveTypes.Int64, Nullable: true}), str: `{"v":-8717895732742165505}`}, + {name: "struct_uint64", dt: arrow.StructOf(arrow.Field{Name: "v", Type: arrow.PrimitiveTypes.Uint64, Nullable: true}), str: `{"v":18428615660272232523}`}, + {name: "list_int64", dt: arrow.ListOf(arrow.PrimitiveTypes.Int64), str: `[-8717895732742165505]`}, + {name: "list_uint64", dt: arrow.ListOf(arrow.PrimitiveTypes.Uint64), str: `[18428615660272232523]`}, + } { + t.Run(tc.name, func(t *testing.T) { + s := NewScalar(tc.dt) + require.NoError(t, s.Set(tc.str)) + require.Equal(t, tc.str, s.String()) + }) + } +} diff --git a/scalar/list.go b/scalar/list.go index 52e30e15d1..1072485a2b 100644 --- a/scalar/list.go +++ b/scalar/list.go @@ -1,7 +1,6 @@ package scalar import ( - "encoding/json" "reflect" "strings" @@ -93,7 +92,7 @@ func (s *List) Set(val any) error { switch value := val.(type) { case string: var x []any - if err := json.Unmarshal([]byte(value), &x); err != nil { + if err := unmarshalJSONWithNumbers([]byte(value), &x); err != nil { return err } length := len(x) @@ -110,7 +109,7 @@ func (s *List) Set(val any) error { case []byte: var x []any - if err := json.Unmarshal(value, &x); err != nil { + if err := unmarshalJSONWithNumbers(value, &x); err != nil { return err } length := len(x) diff --git a/scalar/struct.go b/scalar/struct.go index 1f2cd8b8c6..ed7c04a612 100644 --- a/scalar/struct.go +++ b/scalar/struct.go @@ -59,7 +59,7 @@ func (s *Struct) Set(val any) error { switch value := val.(type) { case string: var x map[string]any - if err := json.Unmarshal([]byte(value), &x); err != nil { + if err := unmarshalJSONWithNumbers([]byte(value), &x); err != nil { return err } for name := range x { @@ -88,7 +88,7 @@ func (s *Struct) Set(val any) error { case []byte: var x map[string]any - if err := json.Unmarshal(value, &x); err != nil { + if err := unmarshalJSONWithNumbers(value, &x); err != nil { return err } s.Value = x diff --git a/scalar/uint.go b/scalar/uint.go index e5c0455f4a..5d29e85704 100644 --- a/scalar/uint.go +++ b/scalar/uint.go @@ -1,6 +1,7 @@ package scalar import ( + "encoding/json" "math" "strconv" @@ -128,6 +129,8 @@ func (s *Uint) Set(val any) error { return &ValidationError{Type: s.DataType(), Msg: "float64 is greater than MaxUint64", Value: value} } return s.Set(uint64(value)) + case json.Number: + return s.Set(string(value)) case string: v, err := strconv.ParseUint(value, 10, 64) if err != nil {