feat: deserialize object from multiple yaml documents (#3862)

* feat: deserialize object from multiple yaml documents

* fix: ignore parse k3s config error

pkg/checker/crictl_checker.go

@@ -74,7 +74,7 @@ func (n *CRICtlChecker) Check(cluster *v2.Cluster, phase string) error {
 	if cfg, err := fileutil.ReadAll(criShimConfig); err != nil {
 		status.Error = fmt.Errorf("read crictl config error: %w", err).Error()
 	} else {
-		cfgMap, _ := yaml.UnmarshalData(cfg)
+		cfgMap, _ := yaml.UnmarshalToMap(cfg)
 		status.Config = map[string]string{}
 		status.Config["ShimSocket"], _, _ = unstructured.NestedString(cfgMap, "image-endpoint")
 		status.Config["CRISocket"], _, _ = unstructured.NestedString(cfgMap, "runtime-endpoint")

pkg/checker/registry_checker.go

@@ -75,7 +75,7 @@ func (n *RegistryChecker) Check(cluster *v2.Cluster, phase string) error {
 	if cfg, err := fileutil.ReadAll(registryConfig); err != nil {
 		status.Error = fmt.Errorf("read registry config error: %w", err).Error()
 	} else {
-		cfgMap, _ := yaml.UnmarshalData(cfg)
+		cfgMap, _ := yaml.UnmarshalToMap(cfg)
 		status.Port, _, _ = unstructured.NestedString(cfgMap, "http", "addr")
 		status.Storage, _, _ = unstructured.NestedString(cfgMap, "storage", "filesystem", "rootdirectory")
 		status.Delete, _, _ = unstructured.NestedBool(cfgMap, "storage", "delete", "enabled")

pkg/clusterfile/pre_process.go

@@ -157,10 +157,7 @@ func (c *ClusterFile) DecodeConfigs(data []byte) error {
 
 func (c *ClusterFile) DecodeRuntimeConfig(data []byte) error {
 	// TODO: handling more types of runtime configuration
-	cfg, err := k3s.ParseConfig(data)
-	if err != nil {
-		return err
-	}
+	cfg, _ := k3s.ParseConfig(data)
 	if cfg != nil {
 		c.runtimeConfig = cfg
 	} else {

pkg/runtime/k3s/config.go

@@ -15,23 +15,19 @@
 package k3s
 
 import (
-	"bufio"
 	"bytes"
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 
-	"github.com/labring/sealos/pkg/utils/iputils"
-
 	"github.com/imdario/mergo"
-	yamlutil "k8s.io/apimachinery/pkg/util/yaml"
 	netutils "k8s.io/utils/net"
-	"sigs.k8s.io/yaml"
 
 	"github.com/labring/sealos/pkg/constants"
 	fileutils "github.com/labring/sealos/pkg/utils/file"
+	"github.com/labring/sealos/pkg/utils/iputils"
 	"github.com/labring/sealos/pkg/utils/logger"
+	"github.com/labring/sealos/pkg/utils/yaml"
 )
 
 var defaultMergeOpts = []func(*mergo.Config){
@@ -170,41 +166,20 @@ func (c *Config) getContainerRuntimeEndpoint() string {
 
 // ParseConfig return nil if data structure is not matched
 func ParseConfig(data []byte) (*Config, error) {
-	d := yamlutil.NewYAMLReader(bufio.NewReader(bytes.NewReader(data)))
-	for {
-		b, err := d.Read()
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			return nil, err
-		}
-		cfg, err := parseConfig(b)
-		if err != nil {
-			return nil, err
-		}
-		if cfg != nil {
-			return cfg, nil
-		}
-	}
-	return nil, nil
-}
-
-func parseConfig(data []byte) (*Config, error) {
-	var c Config
-	if err := yaml.Unmarshal(data, &c); err != nil {
+	var cfg Config
+	if err := yaml.Unmarshal(bytes.NewBuffer(data), &cfg); err != nil {
 		return nil, err
 	}
-	out, err := yaml.Marshal(&c)
+	out, err := yaml.Marshal(&cfg)
 	if err != nil {
 		return nil, err
 	}
-	var m map[string]interface{}
-	if err := yaml.Unmarshal(out, &m); err != nil {
+	isNil, err := yaml.IsNil(out)
+	if err != nil {
 		return nil, err
 	}
-	if len(m) == 0 {
+	if isNil {
 		return nil, nil
 	}
-	return &c, nil
+	return &cfg, nil
 }

pkg/runtime/kubernetes/certs.go

@@ -78,7 +78,7 @@ func (k *KubeadmRuntime) saveNewKubeadmConfig() error {
 		return err
 	}
 	//unmarshal data from configmap
-	obj, err := yaml.UnmarshalData([]byte(data))
+	obj, err := yaml.UnmarshalToMap([]byte(data))
 	if err != nil {
 		return err
 	}

pkg/runtime/kubernetes/kubeadm.go

@@ -209,7 +209,7 @@ func (k *KubeadmRuntime) mergeWithBuiltinKubeadmConfig() error {
 		return err
 	}
 	//unmarshal data from configmap
-	obj, err := yaml.UnmarshalData([]byte(data))
+	obj, err := yaml.UnmarshalToMap([]byte(data))
 	if err != nil {
 		return err
 	}

pkg/utils/yaml/yaml.go

@@ -21,6 +21,9 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"os"
+	"reflect"
+	"strings"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
@@ -29,22 +32,43 @@ import (
 	fileutil "github.com/labring/sealos/pkg/utils/file"
 )
 
-func Unmarshal(path string) (map[string]interface{}, error) {
-	metadata, err := fileutil.ReadAll(path)
-	if err != nil {
-		return nil, err
+const nonStructPointerErrorFmt = "must be a struct pointer, got %T"
+
+func unmarshalStrict(r io.Reader, obj interface{}) (err error) {
+	if obj != nil && reflect.ValueOf(obj).Kind() != reflect.Pointer {
+		return fmt.Errorf(nonStructPointerErrorFmt, obj)
+	}
+	if v := reflect.ValueOf(obj).Elem(); v.Kind() != reflect.Struct {
+		return fmt.Errorf(nonStructPointerErrorFmt, obj)
+	}
+
+	rd := utilyaml.NewYAMLReader(bufio.NewReader(r))
+	for {
+		buf, rerr := rd.Read()
+		if rerr == io.EOF {
+			break
+		}
+		if rerr != nil {
+			return err
+		}
+		if len(bytes.TrimSpace(buf)) == 0 {
+			continue
+		}
+		if err = yaml.UnmarshalStrict(buf, obj); err == nil {
+			return nil
+		}
 	}
-	var data map[string]interface{}
-	err = yaml.Unmarshal(metadata, &data)
 	if err != nil {
-		return nil, err
+		if strings.Contains(err.Error(), "json: unknown field") {
+			err = fmt.Errorf("document do not have corresponding struct %T", obj)
+		}
 	}
-	return data, nil
+	return
 }
 
-func UnmarshalData(metadata []byte) (map[string]interface{}, error) {
+func UnmarshalToMap(buf []byte) (map[string]interface{}, error) {
 	var data map[string]interface{}
-	err := yaml.Unmarshal(metadata, &data)
+	err := yaml.Unmarshal(buf, &data)
 	if err != nil {
 		return nil, err
 	}
@@ -67,24 +91,8 @@ func ToJSON(bs []byte) (jsons []string) {
 	return
 }
 
-func ToYAMLs(bs string) (yamls []string) {
-	buf := bytes.NewBuffer([]byte(bs))
-	reader := utilyaml.NewYAMLReader(bufio.NewReader(buf))
-	for {
-		patch, err := reader.Read()
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			break
-		}
-		patch = bytes.TrimSpace(patch)
-		if len(patch) == 0 {
-			continue
-		}
-		yamls = append(yamls, string(patch))
-	}
-	return
+func Marshal(obj interface{}) ([]byte, error) {
+	return yaml.Marshal(obj)
 }
 
 func MarshalFile(file string, obj ...interface{}) error {
@@ -95,16 +103,27 @@ func MarshalFile(file string, obj ...interface{}) error {
 	return fileutil.WriteFile(file, data)
 }
 
-func UnmarshalFile(file string, obj interface{}) error {
-	metadata, err := fileutil.ReadAll(file)
+func Unmarshal(r io.Reader, obj interface{}) error {
+	return unmarshalStrict(r, obj)
+}
+
+func IsNil(b []byte) (bool, error) {
+	m, err := UnmarshalToMap(b)
 	if err != nil {
-		return err
+		return false, err
 	}
-	err = yaml.Unmarshal(metadata, obj)
+	return len(m) == 0, nil
+}
+
+// UnmarshalFile if there is no content in the file or it contains only spaces,
+// result will be nil, then the given object is not initialized at this time.
+func UnmarshalFile(file string, obj interface{}) error {
+	r, err := os.Open(file)
 	if err != nil {
 		return err
 	}
-	return nil
+	defer r.Close()
+	return unmarshalStrict(r, obj)
 }
 
 func MarshalConfigs(configs ...interface{}) ([]byte, error) {