Skip to content

Commit

Permalink
YAML multi-document support (#32)
Browse files Browse the repository at this point in the history
Add YAML multi-document support
  • Loading branch information
TomWright committed Nov 7, 2020
1 parent 191be70 commit fc77e65
Show file tree
Hide file tree
Showing 9 changed files with 167 additions and 21 deletions.
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ Comparable to [jq](https://github.com/stedolan/jq) / [yq](https://github.com/kis
* [Put](#put)
* [Put Object](#put-object)
* [Supported file types](#supported-file-types)
* [JSON](#json)
* [TOML](#toml)
* [YAML](#yaml)
* [XML](#xml)
* [Selectors](#selectors)
* [Property](#property)
* [Child](#child-elements)
Expand Down Expand Up @@ -334,7 +338,9 @@ Using [gopkg.in/yaml.v2](https://gopkg.in/yaml.v2).
#### Multi-document files
Multi-document files are not supported. This is due to [limitations in the yaml parser used](https://github.com/go-yaml/yaml/tree/v2.3.0#compatibility).
Multi-document files are decoded into an array, with `[0]` being the first document, `[1]` being the second and so on.
Once decoded, you can access them using any of the standard selectors provided by Dasel.
### XML
```bash
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/clbanning/mxj v1.8.5-0.20201012155914-b957cfd48b51 h1:PK4/gQyzsZnkfJdEaWSybHdjsYORx+u34oqN9Sd63Rs=
github.com/clbanning/mxj v1.8.5-0.20201012155914-b957cfd48b51/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
github.com/clbanning/mxj/v2 v2.3.2 h1:DSkU65zfrBHtrggxd54X9pK1z/Lw2OwSW5D8p+x1toE=
github.com/clbanning/mxj/v2 v2.3.2/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
Expand Down Expand Up @@ -103,6 +101,7 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand All @@ -122,6 +121,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
Expand Down
3 changes: 1 addition & 2 deletions internal/command/put.go
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,7 @@ func writeNodeToOutput(opts writeNodeToOutputOpts, cmd *cobra.Command) error {
}
return nil
}

if err := storage.Write(opts.Parser, opts.Node.InterfaceValue(), opts.Writer); err != nil {
if err := storage.Write(opts.Parser, opts.Node.InterfaceValue(), opts.Node.OriginalValue, opts.Writer); err != nil {
return fmt.Errorf("could not write to output file: %w", err)
}

Expand Down
15 changes: 15 additions & 0 deletions internal/command/root_put_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,10 @@ func TestRootCMD_Put_JSON(t *testing.T) {
func TestRootCMD_Put_YAML(t *testing.T) {
t.Run("String", putStringTest(`
id: "x"
name: "Tom"
`, "yaml", "id", "y", `
id: "y"
name: Tom
`, nil))
t.Run("Int", putIntTest(`
id: 123
Expand Down Expand Up @@ -160,6 +162,19 @@ numbers:
rank: 2
- number: three
rank: 3
`, nil))
t.Run("StringInMultiDocument", putStringTest(`
id: "x"
---
id: "y"
---
id: "z"
`, "yaml", "[1].id", "1", `
id: x
---
id: "1"
---
id: z
`, nil))
}

Expand Down
30 changes: 29 additions & 1 deletion internal/storage/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,13 @@ func Load(p Parser, reader io.Reader) (interface{}, error) {
}

// Write writes the value to the given io.Writer.
func Write(p Parser, value interface{}, writer io.Writer) error {
func Write(p Parser, value interface{}, originalValue interface{}, writer io.Writer) error {
switch typed := originalValue.(type) {
case OriginalRequired:
if typed.OriginalRequired() {
value = originalValue
}
}
byteData, err := p.ToBytes(value)
if err != nil {
return fmt.Errorf("could not get byte data for file: %w", err)
Expand All @@ -89,3 +95,25 @@ func Write(p Parser, value interface{}, writer io.Writer) error {
}
return nil
}

// OriginalRequired can be used in conjunction with RealValue to allow parsers to be more intelligent
// with the data they read/write.
type OriginalRequired interface {
// OriginalRequired tells dasel if the parser requires the original value when converting to bytes.
OriginalRequired() bool
}

// RealValue can be used in conjunction with OriginalRequired to allow parsers to be more intelligent
// with the data they read/write.
type RealValue interface {
// RealValue returns the real value that dasel should use when processing data.
RealValue() interface{}
}

type originalRequired struct {
}

// OriginalRequired tells dasel if the parser requires the original value when converting to bytes.
func (d originalRequired) OriginalRequired() bool {
return true
}
6 changes: 3 additions & 3 deletions internal/storage/parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ func (fp *failingReader) Read(_ []byte) (n int, err error) {
func TestWrite(t *testing.T) {
t.Run("Success", func(t *testing.T) {
var buf bytes.Buffer
if err := storage.Write(&storage.JSONParser{}, map[string]interface{}{"name": "Tom"}, &buf); err != nil {
if err := storage.Write(&storage.JSONParser{}, map[string]interface{}{"name": "Tom"}, nil, &buf); err != nil {
t.Errorf("unexpected error: %s", err)
return
}
Expand All @@ -187,14 +187,14 @@ func TestWrite(t *testing.T) {

t.Run("ParserErrHandled", func(t *testing.T) {
var buf bytes.Buffer
if err := storage.Write(&failingParser{}, map[string]interface{}{"name": "Tom"}, &buf); !errors.Is(err, errFailingParserErr) {
if err := storage.Write(&failingParser{}, map[string]interface{}{"name": "Tom"}, nil, &buf); !errors.Is(err, errFailingParserErr) {
t.Errorf("unexpected error: %v", err)
return
}
})

t.Run("WriterErrHandled", func(t *testing.T) {
if err := storage.Write(&storage.JSONParser{}, map[string]interface{}{"name": "Tom"}, &failingWriter{}); !errors.Is(err, errFailingWriterErr) {
if err := storage.Write(&storage.JSONParser{}, map[string]interface{}{"name": "Tom"}, nil, &failingWriter{}); !errors.Is(err, errFailingWriterErr) {
t.Errorf("unexpected error: %v", err)
return
}
Expand Down
72 changes: 67 additions & 5 deletions internal/storage/yaml.go
Original file line number Diff line number Diff line change
@@ -1,24 +1,86 @@
package storage

import (
"bytes"
"fmt"
"gopkg.in/yaml.v2"
"io"
)

// YAMLParser is a Parser implementation to handle yaml files.
type YAMLParser struct {
}

// YAMLSingleDocument represents a decoded single-document YAML file.
type YAMLSingleDocument struct {
originalRequired
Value interface{}
}

// RealValue returns the real value that dasel should use when processing data.
func (d *YAMLSingleDocument) RealValue() interface{} {
return d.Value
}

// YAMLMultiDocument represents a decoded multi-document YAML file.
type YAMLMultiDocument struct {
originalRequired
Values []interface{}
}

// RealValue returns the real value that dasel should use when processing data.
func (d *YAMLMultiDocument) RealValue() interface{} {
return d.Values
}

// FromBytes returns some Data that is represented by the given bytes.
func (p *YAMLParser) FromBytes(byteData []byte) (interface{}, error) {
var data interface{}
if err := yaml.Unmarshal(byteData, &data); err != nil {
return data, fmt.Errorf("could not unmarshal config data: %w", err)
res := make([]interface{}, 0)

decoder := yaml.NewDecoder(bytes.NewBuffer(byteData))

docLoop:
for {
var docData interface{}
if err := decoder.Decode(&docData); err != nil {
if err == io.EOF {
break docLoop
}
return nil, fmt.Errorf("could not unmarshal config data: %w", err)
}
res = append(res, docData)
}
switch len(res) {
case 0:
return nil, nil
case 1:
return &YAMLSingleDocument{Value: res[0]}, nil
default:
return &YAMLMultiDocument{Values: res}, nil
}
return data, nil
}

// ToBytes returns a slice of bytes that represents the given value.
func (p *YAMLParser) ToBytes(value interface{}) ([]byte, error) {
return yaml.Marshal(value)
buffer := new(bytes.Buffer)
encoder := yaml.NewEncoder(buffer)
defer encoder.Close()

switch v := value.(type) {
case *YAMLSingleDocument:
if err := encoder.Encode(v.Value); err != nil {
return nil, fmt.Errorf("could not encode single document: %w", err)
}
case *YAMLMultiDocument:
for index, d := range v.Values {
if err := encoder.Encode(d); err != nil {
return nil, fmt.Errorf("could not encode multi document [%d]: %w", index, err)
}
}
default:
if err := encoder.Encode(v); err != nil {
return nil, fmt.Errorf("could not encode default document type: %w", err)
}
}
return buffer.Bytes(), nil
}
29 changes: 27 additions & 2 deletions internal/storage/yaml_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,33 @@ func TestYAMLParser_FromBytes(t *testing.T) {
t.Errorf("unexpected error: %s", err)
return
}
if !reflect.DeepEqual(yamlMap, got) {
t.Errorf("expected %v, got %v", yamlMap, got)
fmt.Printf("%T", got)
exp := &storage.YAMLSingleDocument{Value: yamlMap}
if !reflect.DeepEqual(exp, got) {
t.Errorf("expected %v, got %v", exp, got)
}
})
t.Run("ValidMultiDocument", func(t *testing.T) {
got, err := (&storage.YAMLParser{}).FromBytes([]byte(`
name: Tom
---
name: Jim
`))
if err != nil {
t.Errorf("unexpected error: %s", err)
return
}
exp := &storage.YAMLMultiDocument{Values: []interface{}{
map[interface{}]interface{}{
"name": "Tom",
},
map[interface{}]interface{}{
"name": "Jim",
},
}}

if !reflect.DeepEqual(exp, got) {
t.Errorf("expected %v, got %v", exp, got)
}
})
t.Run("Invalid", func(t *testing.T) {
Expand Down
21 changes: 16 additions & 5 deletions node.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package dasel

import (
"fmt"
"github.com/tomwright/dasel/internal/storage"
"reflect"
"regexp"
"strconv"
Expand Down Expand Up @@ -32,7 +33,10 @@ type Node struct {
Previous *Node `json:"-"`
// Next is the next node in the chain.
Next *Node `json:"next,omitempty"`

// OriginalValue is the value returned from the parser.
// In most cases this is the same as Value, but is different for thr YAML parser
// as it contains information on the original document.
OriginalValue interface{} `json:"-"`
// Value is the value of the current node.
Value reflect.Value `json:"value"`
// Selector is the selector for the current node.
Expand Down Expand Up @@ -147,11 +151,18 @@ func ParseSelector(selector string) (Selector, error) {

// New returns a new root node with the given value.
func New(value interface{}) *Node {
baseValue := reflect.ValueOf(value)
var baseValue reflect.Value
switch typed := value.(type) {
case storage.RealValue:
baseValue = reflect.ValueOf(typed.RealValue())
default:
baseValue = reflect.ValueOf(value)
}
rootNode := &Node{
Previous: nil,
Next: nil,
Value: baseValue,
Previous: nil,
Next: nil,
OriginalValue: value,
Value: baseValue,
Selector: Selector{
Raw: ".",
Current: ".",
Expand Down

0 comments on commit fc77e65

Please sign in to comment.