Skip to content

Proper usage of yq as a library #2282

Open
@Hammond95

Description

@Hammond95

Hi all, I am trying to use yq in a golang tool, to perform some queries over yaml files and thus perform some validations.

This is the code I've come up with:

package mypakage

import (
	"bytes"
	"fmt"
	"os"
	"strings"

	"github.com/mikefarah/yq/v4/pkg/yqlib"
	logging "gopkg.in/op/go-logging.v1"
	"k8s.io/klog"
)

// ValidateValuesFile reads `values.yaml` and validation file from disk and performs validation.
func ValidateValuesFile(valuesFilePath string, validationFilePath string) error {
	// Read values.yaml into memory
	valuesContent, err := os.ReadFile(valuesFilePath)
	if err != nil {
		return fmt.Errorf("failed to read values.yaml file: %v", err)
	}

	// Read validation file into memory
	validationFileContent, err := os.ReadFile(validationFilePath)
	if err != nil {
		return fmt.Errorf("failed to read validation file: %v", err)
	}

	// Call ValidateValuesFileInternal with valuesContent and validationFileContent in memory
	return ValidateValuesFileInternal(valuesContent, validationFileContent)
}

// ValidateValuesFileWithInMemoryValidation reads `values.yaml` from disk and validates using the provided in-memory validation content.
func ValidateValuesFileWithInMemoryValidation(valuesFilePath string, validationFileContent []byte) error {
	// Read values.yaml into memory
	valuesContent, err := os.ReadFile(valuesFilePath)
	if err != nil {
		return fmt.Errorf("failed to read values.yaml file: %v", err)
	}

	// Call ValidateValuesFileInternal with valuesContent and validationFileContent in memory
	return ValidateValuesFileInternal(valuesContent, validationFileContent)
}

// ValidateValuesFileInternal validates `values.yaml` content against the provided in-memory validation rules.
func ValidateValuesFileInternal(valuesContent []byte, validationFileContent []byte) error {
	var errorMessages []string

	// Initialize ExpressionParser if not already initialized
	// Expression parser is used in existsInYAML function.
	yqlib.InitExpressionParser()
	logging.SetLevel(logging.ERROR, "yq-lib")

	yqEvaluator := yqlib.NewStreamEvaluator()
	if yqEvaluator == nil {
		klog.Fatal("Failed to initialize yq Stream Evaluator!")
	}

	// Process validation rules from the in-memory content
	lines := strings.Split(string(validationFileContent), "\n")
	for _, line := range lines {
		// Skip empty lines
		if strings.TrimSpace(line) == "" {
			continue
		}

		if strings.HasPrefix(line, "#") {
			continue
		}

		klog.Infof("LINE === %s", line)
		parts := strings.Split(line, "|")
		if len(parts) != 3 {
			klog.Warningf("Skipping line '%s' from validation file since it is not valid!", line)
			continue
		}

		oldPath := strings.TrimSpace(parts[0])
		newPath := strings.TrimSpace(parts[1])
		errorMessage := strings.TrimSpace(parts[2])

		// Check if the old path exists in values.yaml
		if existsInYAML(yqEvaluator, valuesContent, oldPath) {
			if len(errorMessage) > 0 {
				errorMessages = append(
					errorMessages,
					fmt.Sprintf("Found values at path %s, please use %s instead. REASON: '%s'.", oldPath, newPath, errorMessage),
				)
			} else {
				errorMessages = append(
					errorMessages,
					fmt.Sprintf("Found values at path %s, please use %s instead.", oldPath, newPath),
				)
			}
		}
	}

	// Return error if validations failed
	if len(errorMessages) > 0 {
		return fmt.Errorf("%s", strings.Join(errorMessages, "\n"))
	}
	return nil
}

// existsInYAML checks if a given path exists in the in-memory YAML content.
func existsInYAML(evaluator yqlib.StreamEvaluator, yamlContent []byte, path string) bool {
	// Parse the expression to create an expression node
	expressionNode, err := yqlib.ExpressionParser.ParseExpression(path)
	if err != nil {
		klog.Infof("Failed to parse expression: %v", err)
		return false
	}

	inputReader := bytes.NewReader(yamlContent)
	// Create a buffer to capture the output
	var outputBuffer bytes.Buffer
	// Create the PrinterWriter with the output buffer
	printerWriter := yqlib.NewSinglePrinterWriter(&outputBuffer)
	// Create the encoder and printer
	encoder := yqlib.NewYamlEncoder(yqlib.YamlPreferences{})
	printer := yqlib.NewPrinter(encoder, printerWriter)
	// Initialize the YAML decoder
	decoder := yqlib.NewYamlDecoder(yqlib.YamlPreferences{})

	// Evaluate the expression against the YAML content
	matches, err := evaluator.Evaluate("", inputReader, expressionNode, printer, decoder)
	if err != nil {
		klog.Warningf("Error evaluating the YAML: %v", err)
		return false
	}

	// If there are any matches, return true; otherwise, false
	return matches > 0
}

Now I've been able to track down the problem to this line:

expressionNode, err := yqlib.ExpressionParser.ParseExpression(path)

which is not working as expected.

I've tried to provide manually some values and these are the results:

path = .foo is evaluated as .foo
path = .foo.bar is evaluated as ., I expect it to be evaluated as .foo.bar
path = .foo.bar.zap is evaluated as ., I expect it to be evaluated as .foo.bar.zap

what am I doing wrong? Could someone help me on this?

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions