From b4ca184108362e1d2bdec096fed6f4fff40083bb Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 29 Aug 2022 15:37:25 +1000 Subject: [PATCH] Added new "c" flag to clobber custom tags when needed --- pkg/yqlib/candidate_node.go | 2 +- pkg/yqlib/doc/operators/assign-update.md | 39 +++++++++++++++++++ .../doc/operators/headers/assign-update.md | 6 ++- .../doc/operators/headers/multiply-merge.md | 1 + pkg/yqlib/doc/operators/multiply-merge.md | 24 ++++++++++++ pkg/yqlib/lexer_participle.go | 17 +++++--- pkg/yqlib/operator_assign.go | 24 +++++++++--- pkg/yqlib/operator_assign_test.go | 17 ++++++++ pkg/yqlib/operator_multiply_test.go | 29 ++++++++++++++ pkg/yqlib/operators.go | 5 ++- release_notes.txt | 5 +++ 11 files changed, 155 insertions(+), 14 deletions(-) diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index 5220b92daa..61675c8a39 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -157,7 +157,7 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP n.Node.Kind = other.Node.Kind // don't clobber custom tags... - if strings.HasPrefix(n.Node.Tag, "!!") || n.Node.Tag == "" { + if prefs.ClobberCustomTags || strings.HasPrefix(n.Node.Tag, "!!") || n.Node.Tag == "" { n.Node.Tag = other.Node.Tag } diff --git a/pkg/yqlib/doc/operators/assign-update.md b/pkg/yqlib/doc/operators/assign-update.md index ef97571d72..af2d80c3c2 100644 --- a/pkg/yqlib/doc/operators/assign-update.md +++ b/pkg/yqlib/doc/operators/assign-update.md @@ -7,6 +7,11 @@ Which will assign the LHS node values to the RHS node values. The RHS expression ### relative form: `|=` This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment. + + +### Flags +- `c` clobber custom tags + {% hint style="warning" %} Note that versions prior to 4.18 require the 'eval/e' command to be specified. @@ -235,3 +240,37 @@ a: - bogs ``` +## Custom types are maintained by default +Given a sample.yml file of: +```yaml +a: !cat meow +b: !dog woof +``` +then +```bash +yq '.a = .b' sample.yml +``` +will output +```yaml +a: !cat woof +b: !dog woof +``` + +## Custom types: clovver +Use the `c` option to clobber custom tags + +Given a sample.yml file of: +```yaml +a: !cat meow +b: !dog woof +``` +then +```bash +yq '.a =c .b' sample.yml +``` +will output +```yaml +a: !dog woof +b: !dog woof +``` + diff --git a/pkg/yqlib/doc/operators/headers/assign-update.md b/pkg/yqlib/doc/operators/headers/assign-update.md index 89a2dc86ba..fc139f3687 100644 --- a/pkg/yqlib/doc/operators/headers/assign-update.md +++ b/pkg/yqlib/doc/operators/headers/assign-update.md @@ -6,4 +6,8 @@ This operator is used to update node values. It can be used in either the: Which will assign the LHS node values to the RHS node values. The RHS expression is run against the matching nodes in the pipeline. ### relative form: `|=` -This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment. \ No newline at end of file +This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment. + + +### Flags +- `c` clobber custom tags diff --git a/pkg/yqlib/doc/operators/headers/multiply-merge.md b/pkg/yqlib/doc/operators/headers/multiply-merge.md index 26e0a57bfd..1dad37e854 100644 --- a/pkg/yqlib/doc/operators/headers/multiply-merge.md +++ b/pkg/yqlib/doc/operators/headers/multiply-merge.md @@ -14,6 +14,7 @@ You can control how objects are merged by using one or more of the following fla - `d` deeply merge arrays - `?` only merge _existing_ fields - `n` only merge _new_ fields +- `c` clobber custom tags ### Merge two files together diff --git a/pkg/yqlib/doc/operators/multiply-merge.md b/pkg/yqlib/doc/operators/multiply-merge.md index 9f04daed8b..c203e3574a 100644 --- a/pkg/yqlib/doc/operators/multiply-merge.md +++ b/pkg/yqlib/doc/operators/multiply-merge.md @@ -14,6 +14,7 @@ You can control how objects are merged by using one or more of the following fla - `d` deeply merge arrays - `?` only merge _existing_ fields - `n` only merge _new_ fields +- `c` clobber custom tags ### Merge two files together @@ -481,3 +482,26 @@ b: !goat dog: woof ``` +## Custom types: clobber tags +Use the `c` option to clobber custom tags. Note that the second tag is now used + +Given a sample.yml file of: +```yaml +a: !horse + cat: meow +b: !goat + dog: woof +``` +then +```bash +yq '.a *=c .b' sample.yml +``` +will output +```yaml +a: !goat + cat: meow + dog: woof +b: !goat + dog: woof +``` + diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 8945f5eb5c..a289aa377e 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -182,8 +182,8 @@ var participleYqRules = []*participleYqRule{ {"GreaterThan", `\s*>\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: true}), 0}, {"LessThan", `\s*<\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: false}), 0}, - {"AssignRelative", `\|=`, assignOpToken(true), 0}, - {"Assign", `=`, assignOpToken(false), 0}, + {"AssignRelative", `\|=[c]*`, assignOpToken(true), 0}, + {"Assign", `=[c]*`, assignOpToken(false), 0}, {`whitespace`, `[ \t\n]+`, nil, 0}, @@ -194,8 +194,8 @@ var participleYqRules = []*participleYqRule{ {"Union", `,`, opToken(unionOpType), 0}, - {"MultiplyAssign", `\*=[\+|\?dn]*`, multiplyWithPrefs(multiplyAssignOpType), 0}, - {"Multiply", `\*[\+|\?dn]*`, multiplyWithPrefs(multiplyOpType), 0}, + {"MultiplyAssign", `\*=[\+|\?cdn]*`, multiplyWithPrefs(multiplyAssignOpType), 0}, + {"Multiply", `\*[\+|\?cdn]*`, multiplyWithPrefs(multiplyOpType), 0}, {"AddAssign", `\+=`, opToken(addAssignOpType), 0}, {"Add", `\+`, opToken(addOpType), 0}, @@ -317,6 +317,9 @@ func assignOpToken(updateAssign bool) yqAction { log.Debug("assignOpToken %v", rawToken.Value) value := rawToken.Value prefs := assignPreferences{DontOverWriteAnchor: true} + if strings.Contains(value, "c") { + prefs.ClobberCustomTags = true + } op := &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, UpdateAssign: updateAssign, Preferences: prefs} return &token{TokenType: operationToken, Operation: op}, nil } @@ -387,6 +390,7 @@ func envSubstWithOptions() yqAction { func multiplyWithPrefs(op *operationType) yqAction { return func(rawToken lexer.Token) (*token, error) { prefs := multiplyPreferences{} + prefs.AssignPrefs = assignPreferences{} options := rawToken.Value if strings.Contains(options, "+") { prefs.AppendArrays = true @@ -395,11 +399,14 @@ func multiplyWithPrefs(op *operationType) yqAction { prefs.TraversePrefs = traversePreferences{DontAutoCreate: true} } if strings.Contains(options, "n") { - prefs.AssignPrefs = assignPreferences{OnlyWriteNull: true} + prefs.AssignPrefs.OnlyWriteNull = true } if strings.Contains(options, "d") { prefs.DeepMergeArrays = true } + if strings.Contains(options, "c") { + prefs.AssignPrefs.ClobberCustomTags = true + } prefs.TraversePrefs.DontFollowAlias = true op := &Operation{OperationType: op, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs} return &token{TokenType: operationToken, Operation: op}, nil diff --git a/pkg/yqlib/operator_assign.go b/pkg/yqlib/operator_assign.go index 2e8c0e1f50..f21f4522a5 100644 --- a/pkg/yqlib/operator_assign.go +++ b/pkg/yqlib/operator_assign.go @@ -3,6 +3,7 @@ package yqlib type assignPreferences struct { DontOverWriteAnchor bool OnlyWriteNull bool + ClobberCustomTags bool } func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation { @@ -15,18 +16,29 @@ func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation { } } +// they way *= (multipleAssign) is handled, we set the multiplePrefs +// on the node, not assignPrefs. Long story. +func getAssignPreferences(preferences interface{}) assignPreferences { + prefs := assignPreferences{} + + switch typedPref := preferences.(type) { + case assignPreferences: + prefs = typedPref + case multiplyPreferences: + prefs = typedPref.AssignPrefs + } + return prefs +} + func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) if err != nil { return Context{}, err } - prefs := assignPreferences{} - // they way *= (multipleAssign) is handled, we set the multiplePrefs - // on the node, not assignPrefs. Long story. - if p, ok := expressionNode.Operation.Preferences.(assignPreferences); ok { - prefs = p - } + prefs := getAssignPreferences(expressionNode.Operation.Preferences) + + log.Debug("assignUpdateOperator prefs: %v", prefs) if !expressionNode.Operation.UpdateAssign { // this works because we already ran against LHS with an editable context. diff --git a/pkg/yqlib/operator_assign_test.go b/pkg/yqlib/operator_assign_test.go index 052be8b252..4a7cd2088f 100644 --- a/pkg/yqlib/operator_assign_test.go +++ b/pkg/yqlib/operator_assign_test.go @@ -208,6 +208,23 @@ var assignOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::a:\n b:\n - null\n - c: bogs\n", }, }, + { + description: "Custom types are maintained by default", + document: "a: !cat meow\nb: !dog woof", + expression: `.a = .b`, + expected: []string{ + "D0, P[], (doc)::a: !cat woof\nb: !dog woof\n", + }, + }, + { + description: "Custom types: clovver", + subdescription: "Use the `c` option to clobber custom tags", + document: "a: !cat meow\nb: !dog woof", + expression: `.a =c .b`, + expected: []string{ + "D0, P[], (doc)::a: !dog woof\nb: !dog woof\n", + }, + }, } func TestAssignOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index 9924150f4f..8a16a411c5 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -523,6 +523,35 @@ var multiplyOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::a: !horse {cat: meow, dog: woof}\nb: !goat {dog: woof}\n", }, }, + { + description: "Custom types: clobber tags", + subdescription: "Use the `c` option to clobber custom tags. Note that the second tag is now used", + document: "a: !horse {cat: meow}\nb: !goat {dog: woof}", + expression: ".a *=c .b", + expected: []string{ + "D0, P[], (doc)::a: !goat {cat: meow, dog: woof}\nb: !goat {dog: woof}\n", + }, + }, + { + skipDoc: true, + description: "Custom types: clobber tags - *=", + subdescription: "Use the `c` option to clobber custom tags - on both the `=` and `*` operator. Note that the second tag is now used", + document: "a: !horse {cat: meow}\nb: !goat {dog: woof}", + expression: ".a =c .a *c .b", + expected: []string{ + "D0, P[], (doc)::a: !goat {cat: meow, dog: woof}\nb: !goat {dog: woof}\n", + }, + }, + { + skipDoc: true, + description: "Custom types: dont clobber tags - *=", + subdescription: "Use the `c` option to clobber custom tags - on both the `=` and `*` operator. Note that the second tag is now used", + document: "a: !horse {cat: meow}\nb: !goat {dog: woof}", + expression: ".a *= .b", + expected: []string{ + "D0, P[], (doc)::a: !horse {cat: meow, dog: woof}\nb: !goat {dog: woof}\n", + }, + }, { skipDoc: true, description: "Custom types: that are really maps", diff --git a/pkg/yqlib/operators.go b/pkg/yqlib/operators.go index 1a70db6ab8..c4b79f7a31 100644 --- a/pkg/yqlib/operators.go +++ b/pkg/yqlib/operators.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/jinzhu/copier" + logging "gopkg.in/op/go-logging.v1" "gopkg.in/yaml.v3" ) @@ -83,8 +84,10 @@ func resultsForRHS(d *dataTreeNavigator, context Context, lhsCandidate *Candidat } for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() { - log.Debugf("Applying calc") rhsCandidate := rightEl.Value.(*CandidateNode) + if !log.IsEnabledFor(logging.DEBUG) { + log.Debugf("Applying lhs: %v, rhsCandidate, %v", NodeToString(lhsCandidate), NodeToString(rhsCandidate)) + } resultCandidate, err := prefs.Calculation(d, context, lhsCandidate, rhsCandidate) if err != nil { return err diff --git a/release_notes.txt b/release_notes.txt index f8b6b2fc89..94562ffa8d 100644 --- a/release_notes.txt +++ b/release_notes.txt @@ -1,3 +1,8 @@ +4.27.3: + - Added new 'c' merge and assign flag that clobbers custom tags + - Bumped go dependency to fix CVE (#1316) + - Updated dependencies + 4.27.2: - Fixed JSON decoder to maintain object key order.