Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add yaml util to unmarshal numbers into int/float #95836

Merged
merged 1 commit into from Oct 26, 2020
Merged

Add yaml util to unmarshal numbers into int/float #95836

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
33 changes: 18 additions & 15 deletions staging/src/k8s.io/apimachinery/pkg/util/json/json.go
View file
Open in desktop
Expand Up @@ -39,7 +39,8 @@ func Marshal(v interface{}) ([]byte, error) {
const maxDepth = 10000

// Unmarshal unmarshals the given data
// If v is a *map[string]interface{}, numbers are converted to int64 or float64
// If v is a *map[string]interface{}, *[]interface{}, or *interface{} numbers
// are converted to int64 or float64
func Unmarshal(data []byte, v interface{}) error {
switch v := v.(type) {
case *map[string]interface{}:
Expand All @@ -52,7 +53,7 @@ func Unmarshal(data []byte, v interface{}) error {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return convertMapNumbers(*v, 0)
return ConvertMapNumbers(*v, 0)

case *[]interface{}:
// Build a decoder from the given data
Expand All @@ -64,7 +65,7 @@ func Unmarshal(data []byte, v interface{}) error {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return convertSliceNumbers(*v, 0)
return ConvertSliceNumbers(*v, 0)

case *interface{}:
// Build a decoder from the given data
Expand All @@ -76,29 +77,31 @@ func Unmarshal(data []byte, v interface{}) error {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return convertInterfaceNumbers(v, 0)
return ConvertInterfaceNumbers(v, 0)

default:
return json.Unmarshal(data, v)
}
}

func convertInterfaceNumbers(v *interface{}, depth int) error {
// ConvertInterfaceNumbers converts any json.Number values to int64 or float64.
// Values which are map[string]interface{} or []interface{} are recursively visited
func ConvertInterfaceNumbers(v *interface{}, depth int) error {
var err error
switch v2 := (*v).(type) {
case json.Number:
*v, err = convertNumber(v2)
case map[string]interface{}:
err = convertMapNumbers(v2, depth+1)
err = ConvertMapNumbers(v2, depth+1)
case []interface{}:
err = convertSliceNumbers(v2, depth+1)
err = ConvertSliceNumbers(v2, depth+1)
}
return err
}

// convertMapNumbers traverses the map, converting any json.Number values to int64 or float64.
// ConvertMapNumbers traverses the map, converting any json.Number values to int64 or float64.
// values which are map[string]interface{} or []interface{} are recursively visited
func convertMapNumbers(m map[string]interface{}, depth int) error {
func ConvertMapNumbers(m map[string]interface{}, depth int) error {
if depth > maxDepth {
return fmt.Errorf("exceeded max depth of %d", maxDepth)
}
Expand All @@ -109,9 +112,9 @@ func convertMapNumbers(m map[string]interface{}, depth int) error {
case json.Number:
m[k], err = convertNumber(v)
case map[string]interface{}:
err = convertMapNumbers(v, depth+1)
err = ConvertMapNumbers(v, depth+1)
case []interface{}:
err = convertSliceNumbers(v, depth+1)
err = ConvertSliceNumbers(v, depth+1)
}
if err != nil {
return err
Expand All @@ -120,9 +123,9 @@ func convertMapNumbers(m map[string]interface{}, depth int) error {
return nil
}

// convertSliceNumbers traverses the slice, converting any json.Number values to int64 or float64.
// ConvertSliceNumbers traverses the slice, converting any json.Number values to int64 or float64.
// values which are map[string]interface{} or []interface{} are recursively visited
func convertSliceNumbers(s []interface{}, depth int) error {
func ConvertSliceNumbers(s []interface{}, depth int) error {
if depth > maxDepth {
return fmt.Errorf("exceeded max depth of %d", maxDepth)
}
Expand All @@ -133,9 +136,9 @@ func convertSliceNumbers(s []interface{}, depth int) error {
case json.Number:
s[i], err = convertNumber(v)
case map[string]interface{}:
err = convertMapNumbers(v, depth+1)
err = ConvertMapNumbers(v, depth+1)
case []interface{}:
err = convertSliceNumbers(v, depth+1)
err = ConvertSliceNumbers(v, depth+1)
}
if err != nil {
return err
Expand Down
1 change: 1 addition & 0 deletions staging/src/k8s.io/apimachinery/pkg/util/yaml/BUILD
View file
Open in desktop
Expand Up @@ -18,6 +18,7 @@ go_library(
importmap = "k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/yaml",
importpath = "k8s.io/apimachinery/pkg/util/yaml",
deps = [
"//staging/src/k8s.io/apimachinery/pkg/util/json:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
"//vendor/sigs.k8s.io/yaml:go_default_library",
],
Expand Down
31 changes: 31 additions & 0 deletions staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go
View file
Open in desktop
Expand Up @@ -26,10 +26,41 @@ import (
"strings"
"unicode"

jsonutil "k8s.io/apimachinery/pkg/util/json"

"k8s.io/klog/v2"
"sigs.k8s.io/yaml"
)

// Unmarshal unmarshals the given data
// If v is a *map[string]interface{}, *[]interface{}, or *interface{} numbers
// are converted to int64 or float64
func Unmarshal(data []byte, v interface{}) error {
preserveIntFloat := func(d *json.Decoder) *json.Decoder {
d.UseNumber()
liggitt marked this conversation as resolved.
Show resolved Hide resolved
return d
}
switch v := v.(type) {
case *map[string]interface{}:
if err := yaml.Unmarshal(data, v, preserveIntFloat); err != nil {
return err
}
return jsonutil.ConvertMapNumbers(*v, 0)
case *[]interface{}:
if err := yaml.Unmarshal(data, v, preserveIntFloat); err != nil {
return err
}
return jsonutil.ConvertSliceNumbers(*v, 0)
case *interface{}:
if err := yaml.Unmarshal(data, v, preserveIntFloat); err != nil {
return err
}
return jsonutil.ConvertInterfaceNumbers(v, 0)
default:
return yaml.Unmarshal(data, v)
}
}

// ToJSON converts a single YAML document into a JSON document
// or returns an error. If the document appears to be JSON the
// YAML decoding path is not used (so that error messages are
Expand Down
41 changes: 41 additions & 0 deletions staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder_test.go
View file
Open in desktop
Expand Up @@ -430,3 +430,44 @@ stuff: 1
t.Fatalf("expected %q to be of type YAMLSyntaxError", err.Error())
}
}

func TestUnmarshal(t *testing.T) {
mapWithIntegerBytes := []byte(`replicas: 1`)
mapWithInteger := make(map[string]interface{})
if err := Unmarshal(mapWithIntegerBytes, &mapWithInteger); err != nil {
t.Fatalf("unexpected error unmarshaling yaml: %v", err)
}
if _, ok := mapWithInteger["replicas"].(int64); !ok {
t.Fatalf(`Expected number in map to be int64 but got "%T"`, mapWithInteger["replicas"])
}

sliceWithIntegerBytes := []byte(`- 1`)
var sliceWithInteger []interface{}
if err := Unmarshal(sliceWithIntegerBytes, &sliceWithInteger); err != nil {
t.Fatalf("unexpected error unmarshaling yaml: %v", err)
}
if _, ok := sliceWithInteger[0].(int64); !ok {
t.Fatalf(`Expected number in slice to be int64 but got "%T"`, sliceWithInteger[0])
}

integerBytes := []byte(`1`)
var integer interface{}
if err := Unmarshal(integerBytes, &integer); err != nil {
t.Fatalf("unexpected error unmarshaling yaml: %v", err)
}
if _, ok := integer.(int64); !ok {
t.Fatalf(`Expected number to be int64 but got "%T"`, integer)
}

otherTypeBytes := []byte(`123: 2`)
otherType := make(map[int]interface{})
if err := Unmarshal(otherTypeBytes, &otherType); err != nil {
t.Fatalf("unexpected error unmarshaling yaml: %v", err)
}
if _, ok := otherType[123].(int64); ok {
t.Fatalf(`Expected number not to be converted to int64`)
}
if _, ok := otherType[123].(float64); !ok {
t.Fatalf(`Expected number to be float64 but got "%T"`, otherType[123])
}
}
2 changes: 1 addition & 1 deletion staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD
View file
Open in desktop
Expand Up @@ -87,6 +87,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/audit:go_default_library",
Expand All @@ -108,7 +109,6 @@ go_library(
"//vendor/google.golang.org/grpc/status:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
"//vendor/k8s.io/utils/trace:go_default_library",
"//vendor/sigs.k8s.io/yaml:go_default_library",
],
)

Expand Down
2 changes: 1 addition & 1 deletion staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go
View file
Open in desktop
Expand Up @@ -38,6 +38,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authorization/authorizer"
Expand All @@ -49,7 +50,6 @@ import (
"k8s.io/apiserver/pkg/util/dryrun"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utiltrace "k8s.io/utils/trace"
"sigs.k8s.io/yaml"
)

const (
Expand Down