Proper usage of yq as a library

[Hammond95]

Hi all, I am trying to use yq in a golang tool, to perform some queries over yaml files and thus perform some validations.

This is the code I've come up with:

package mypakage

import (
	"bytes"
	"fmt"
	"os"
	"strings"

	"github.com/mikefarah/yq/v4/pkg/yqlib"
	logging "gopkg.in/op/go-logging.v1"
	"k8s.io/klog"
)

// ValidateValuesFile reads `values.yaml` and validation file from disk and performs validation.
func ValidateValuesFile(valuesFilePath string, validationFilePath string) error {
	// Read values.yaml into memory
	valuesContent, err := os.ReadFile(valuesFilePath)
	if err != nil {
		return fmt.Errorf("failed to read values.yaml file: %v", err)
	}

	// Read validation file into memory
	validationFileContent, err := os.ReadFile(validationFilePath)
	if err != nil {
		return fmt.Errorf("failed to read validation file: %v", err)
	}

	// Call ValidateValuesFileInternal with valuesContent and validationFileContent in memory
	return ValidateValuesFileInternal(valuesContent, validationFileContent)
}

// ValidateValuesFileWithInMemoryValidation reads `values.yaml` from disk and validates using the provided in-memory validation content.
func ValidateValuesFileWithInMemoryValidation(valuesFilePath string, validationFileContent []byte) error {
	// Read values.yaml into memory
	valuesContent, err := os.ReadFile(valuesFilePath)
	if err != nil {
		return fmt.Errorf("failed to read values.yaml file: %v", err)
	}

	// Call ValidateValuesFileInternal with valuesContent and validationFileContent in memory
	return ValidateValuesFileInternal(valuesContent, validationFileContent)
}

// ValidateValuesFileInternal validates `values.yaml` content against the provided in-memory validation rules.
func ValidateValuesFileInternal(valuesContent []byte, validationFileContent []byte) error {
	var errorMessages []string

	// Initialize ExpressionParser if not already initialized
	// Expression parser is used in existsInYAML function.
	yqlib.InitExpressionParser()
	logging.SetLevel(logging.ERROR, "yq-lib")

	yqEvaluator := yqlib.NewStreamEvaluator()
	if yqEvaluator == nil {
		klog.Fatal("Failed to initialize yq Stream Evaluator!")
	}

	// Process validation rules from the in-memory content
	lines := strings.Split(string(validationFileContent), "\n")
	for _, line := range lines {
		// Skip empty lines
		if strings.TrimSpace(line) == "" {
			continue
		}

		if strings.HasPrefix(line, "#") {
			continue
		}

		klog.Infof("LINE === %s", line)
		parts := strings.Split(line, "|")
		if len(parts) != 3 {
			klog.Warningf("Skipping line '%s' from validation file since it is not valid!", line)
			continue
		}

		oldPath := strings.TrimSpace(parts[0])
		newPath := strings.TrimSpace(parts[1])
		errorMessage := strings.TrimSpace(parts[2])

		// Check if the old path exists in values.yaml
		if existsInYAML(yqEvaluator, valuesContent, oldPath) {
			if len(errorMessage) > 0 {
				errorMessages = append(
					errorMessages,
					fmt.Sprintf("Found values at path %s, please use %s instead. REASON: '%s'.", oldPath, newPath, errorMessage),
				)
			} else {
				errorMessages = append(
					errorMessages,
					fmt.Sprintf("Found values at path %s, please use %s instead.", oldPath, newPath),
				)
			}
		}
	}

	// Return error if validations failed
	if len(errorMessages) > 0 {
		return fmt.Errorf("%s", strings.Join(errorMessages, "\n"))
	}
	return nil
}

// existsInYAML checks if a given path exists in the in-memory YAML content.
func existsInYAML(evaluator yqlib.StreamEvaluator, yamlContent []byte, path string) bool {
	// Parse the expression to create an expression node
	expressionNode, err := yqlib.ExpressionParser.ParseExpression(path)
	if err != nil {
		klog.Infof("Failed to parse expression: %v", err)
		return false
	}

	inputReader := bytes.NewReader(yamlContent)
	// Create a buffer to capture the output
	var outputBuffer bytes.Buffer
	// Create the PrinterWriter with the output buffer
	printerWriter := yqlib.NewSinglePrinterWriter(&outputBuffer)
	// Create the encoder and printer
	encoder := yqlib.NewYamlEncoder(yqlib.YamlPreferences{})
	printer := yqlib.NewPrinter(encoder, printerWriter)
	// Initialize the YAML decoder
	decoder := yqlib.NewYamlDecoder(yqlib.YamlPreferences{})

	// Evaluate the expression against the YAML content
	matches, err := evaluator.Evaluate("", inputReader, expressionNode, printer, decoder)
	if err != nil {
		klog.Warningf("Error evaluating the YAML: %v", err)
		return false
	}

	// If there are any matches, return true; otherwise, false
	return matches > 0
}

Now I've been able to track down the problem to this line:

expressionNode, err := yqlib.ExpressionParser.ParseExpression(path)

which is not working as expected.

I've tried to provide manually some values and these are the results:

path = .foo is evaluated as .foo
path = .foo.bar is evaluated as ., I expect it to be evaluated as .foo.bar
path = .foo.bar.zap is evaluated as ., I expect it to be evaluated as .foo.bar.zap

what am I doing wrong? Could someone help me on this?

[Hammond95]

@mikefarah sorry to tag you directly here, but maybe this could be quite easy for you to answer. 🙏