From c1b333b2c25f45b9265cf229611b2253fb424b35 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Tue, 18 Oct 2022 14:15:25 +1100
Subject: [PATCH 01/10] Added xml support for proc instructions
---
examples/mike.xml | 20 ++++++--------------
pkg/yqlib/decoder_xml.go | 8 ++++++--
pkg/yqlib/encoder_xml.go | 38 +++++++++++++++++++++++++++++---------
3 files changed, 41 insertions(+), 25 deletions(-)
diff --git a/examples/mike.xml b/examples/mike.xml
index 94773b8633..75a651933e 100644
--- a/examples/mike.xml
+++ b/examples/mike.xml
@@ -1,14 +1,6 @@
-<!-- before cat -->
-<cat>
- <!-- in cat before -->
- <x>3<!-- multi
-line comment
-for x --></x>
- <y>
- <!-- in y before -->
- <d><!-- in d before -->4<!-- in d after --></d>
- <!-- in y after -->
- </y>
- <!-- in_cat_after -->
-</cat>
-<!-- after cat -->
\ No newline at end of file
+<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<apple>
+Cats are cgreat
+<b>things</b>
+</apple>
\ No newline at end of file
diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go
index d053f6caba..7ff1bcee6a 100644
--- a/pkg/yqlib/decoder_xml.go
+++ b/pkg/yqlib/decoder_xml.go
@@ -15,6 +15,8 @@ type xmlDecoder struct {
reader io.Reader
readAnything bool
attributePrefix string
+ directivePrefix string
+ procInstPrefix string
contentName string
strictMode bool
keepNamespace bool
@@ -33,6 +35,8 @@ func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool,
strictMode: strictMode,
keepNamespace: keepNamespace,
useRawToken: useRawToken,
+ directivePrefix: "_directive_",
+ procInstPrefix: "_procInst_",
}
}
@@ -86,9 +90,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
}
- // if i == len(n.Children)-1 {
labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
- // }
log.Debug("len of children in %v is %v", label, len(children))
if len(children) > 1 {
@@ -282,6 +284,8 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
elem.n.HeadComment = joinFilter([]string{elem.n.HeadComment, commentStr})
}
+ case xml.ProcInst:
+ elem.n.AddChild(dec.procInstPrefix+se.Target, &xmlNode{Data: string(se.Inst)})
}
}
diff --git a/pkg/yqlib/encoder_xml.go b/pkg/yqlib/encoder_xml.go
index b84b59de39..39cf1bfc1e 100644
--- a/pkg/yqlib/encoder_xml.go
+++ b/pkg/yqlib/encoder_xml.go
@@ -15,6 +15,8 @@ type xmlEncoder struct {
attributePrefix string
contentName string
indentString string
+ directivePrefix string
+ procInstPrefix string
}
func NewXMLEncoder(indent int, attributePrefix string, contentName string) Encoder {
@@ -23,7 +25,7 @@ func NewXMLEncoder(indent int, attributePrefix string, contentName string) Encod
for index := 0; index < indent; index++ {
indentString = indentString + " "
}
- return &xmlEncoder{attributePrefix, contentName, indentString}
+ return &xmlEncoder{attributePrefix, contentName, indentString, "_directive_", "_procInst_"}
}
func (e *xmlEncoder) CanHandleAliases() bool {
@@ -44,7 +46,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
switch node.Kind {
case yaml.MappingNode:
- err := e.encodeTopLevelMap(encoder, node)
+ err := e.encodeTopLevelMap(encoder, node, writer)
if err != nil {
return err
}
@@ -53,7 +55,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
if err != nil {
return err
}
- err = e.encodeTopLevelMap(encoder, unwrapDoc(node))
+ err = e.encodeTopLevelMap(encoder, unwrapDoc(node), writer)
if err != nil {
return err
}
@@ -76,7 +78,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
}
-func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) error {
+func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node, writer io.Writer) error {
err := e.encodeComment(encoder, headAndLineComment(node))
if err != nil {
return err
@@ -92,11 +94,23 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) er
return err
}
- log.Debugf("recursing")
+ if strings.HasPrefix(key.Value, e.procInstPrefix) && key.Value != e.contentName {
+ name := strings.Replace(key.Value, e.procInstPrefix, "", 1)
+ procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
+ if err := encoder.EncodeToken(procInst); err != nil {
+ return err
+ }
+ if _, err := writer.Write([]byte("\n")); err != nil {
+ log.Warning("Unable to write newline, skipping: %w", err)
+ }
+ } else {
- err = e.doEncode(encoder, value, start)
- if err != nil {
- return err
+ log.Debugf("recursing")
+
+ err = e.doEncode(encoder, value, start)
+ if err != nil {
+ return err
+ }
}
err = e.encodeComment(encoder, footComment(key))
if err != nil {
@@ -212,8 +226,14 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.
if err != nil {
return err
}
+ if strings.HasPrefix(key.Value, e.procInstPrefix) && key.Value != e.contentName {
+ name := strings.Replace(key.Value, e.procInstPrefix, "", 1)
+ procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
+ if err := encoder.EncodeToken(procInst); err != nil {
+ return err
+ }
- if !strings.HasPrefix(key.Value, e.attributePrefix) && key.Value != e.contentName {
+ } else if !strings.HasPrefix(key.Value, e.attributePrefix) && key.Value != e.contentName {
start := xml.StartElement{Name: xml.Name{Local: key.Value}}
err := e.doEncode(encoder, value, start)
if err != nil {
From 6bbab2a66442e05f0edfdf3cda5a9ac24bacc240 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Tue, 18 Oct 2022 14:33:57 +1100
Subject: [PATCH 02/10] Can handle xml directives
---
examples/mike.xml | 3 ++-
pkg/yqlib/decoder_xml.go | 6 ++++--
pkg/yqlib/encoder_xml.go | 33 ++++++++++++++++++++++++---------
3 files changed, 30 insertions(+), 12 deletions(-)
diff --git a/examples/mike.xml b/examples/mike.xml
index 75a651933e..4543a10688 100644
--- a/examples/mike.xml
+++ b/examples/mike.xml
@@ -1,6 +1,7 @@
<?xml version="1.0"?>
<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
<apple>
-Cats are cgreat
+<?coolioo version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
<b>things</b>
</apple>
\ No newline at end of file
diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go
index 7ff1bcee6a..4a2a0c82b0 100644
--- a/pkg/yqlib/decoder_xml.go
+++ b/pkg/yqlib/decoder_xml.go
@@ -15,7 +15,7 @@ type xmlDecoder struct {
reader io.Reader
readAnything bool
attributePrefix string
- directivePrefix string
+ directiveName string
procInstPrefix string
contentName string
strictMode bool
@@ -35,7 +35,7 @@ func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool,
strictMode: strictMode,
keepNamespace: keepNamespace,
useRawToken: useRawToken,
- directivePrefix: "_directive_",
+ directiveName: "_directive_",
procInstPrefix: "_procInst_",
}
}
@@ -286,6 +286,8 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
case xml.ProcInst:
elem.n.AddChild(dec.procInstPrefix+se.Target, &xmlNode{Data: string(se.Inst)})
+ case xml.Directive:
+ elem.n.AddChild(dec.directiveName, &xmlNode{Data: string(se)})
}
}
diff --git a/pkg/yqlib/encoder_xml.go b/pkg/yqlib/encoder_xml.go
index 39cf1bfc1e..ecb6a431ae 100644
--- a/pkg/yqlib/encoder_xml.go
+++ b/pkg/yqlib/encoder_xml.go
@@ -15,8 +15,9 @@ type xmlEncoder struct {
attributePrefix string
contentName string
indentString string
- directivePrefix string
+ directiveName string
procInstPrefix string
+ writer io.Writer
}
func NewXMLEncoder(indent int, attributePrefix string, contentName string) Encoder {
@@ -25,7 +26,7 @@ func NewXMLEncoder(indent int, attributePrefix string, contentName string) Encod
for index := 0; index < indent; index++ {
indentString = indentString + " "
}
- return &xmlEncoder{attributePrefix, contentName, indentString, "_directive_", "_procInst_"}
+ return &xmlEncoder{attributePrefix, contentName, indentString, "_directive_", "_procInst_", nil}
}
func (e *xmlEncoder) CanHandleAliases() bool {
@@ -42,11 +43,13 @@ func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error
func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
encoder := xml.NewEncoder(writer)
+ // hack so we can manually add newlines to procInst and directives
+ e.writer = writer
encoder.Indent("", e.indentString)
switch node.Kind {
case yaml.MappingNode:
- err := e.encodeTopLevelMap(encoder, node, writer)
+ err := e.encodeTopLevelMap(encoder, node)
if err != nil {
return err
}
@@ -55,7 +58,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
if err != nil {
return err
}
- err = e.encodeTopLevelMap(encoder, unwrapDoc(node), writer)
+ err = e.encodeTopLevelMap(encoder, unwrapDoc(node))
if err != nil {
return err
}
@@ -78,7 +81,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
}
-func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node, writer io.Writer) error {
+func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) error {
err := e.encodeComment(encoder, headAndLineComment(node))
if err != nil {
return err
@@ -94,13 +97,21 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node, wr
return err
}
- if strings.HasPrefix(key.Value, e.procInstPrefix) && key.Value != e.contentName {
+ if strings.HasPrefix(key.Value, e.procInstPrefix) {
name := strings.Replace(key.Value, e.procInstPrefix, "", 1)
procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
if err := encoder.EncodeToken(procInst); err != nil {
return err
}
- if _, err := writer.Write([]byte("\n")); err != nil {
+ if _, err := e.writer.Write([]byte("\n")); err != nil {
+ log.Warning("Unable to write newline, skipping: %w", err)
+ }
+ } else if key.Value == e.directiveName {
+ var directive xml.Directive = []byte(value.Value)
+ if err := encoder.EncodeToken(directive); err != nil {
+ return err
+ }
+ if _, err := e.writer.Write([]byte("\n")); err != nil {
log.Warning("Unable to write newline, skipping: %w", err)
}
} else {
@@ -226,13 +237,17 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.
if err != nil {
return err
}
- if strings.HasPrefix(key.Value, e.procInstPrefix) && key.Value != e.contentName {
+ if strings.HasPrefix(key.Value, e.procInstPrefix) {
name := strings.Replace(key.Value, e.procInstPrefix, "", 1)
procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
if err := encoder.EncodeToken(procInst); err != nil {
return err
}
-
+ } else if key.Value == e.directiveName {
+ var directive xml.Directive = []byte(value.Value)
+ if err := encoder.EncodeToken(directive); err != nil {
+ return err
+ }
} else if !strings.HasPrefix(key.Value, e.attributePrefix) && key.Value != e.contentName {
start := xml.StartElement{Name: xml.Name{Local: key.Value}}
err := e.doEncode(encoder, value, start)
From 2e9c91f8a1344e91766ae09a5ae2326572a0b6be Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Tue, 18 Oct 2022 14:56:50 +1100
Subject: [PATCH 03/10] wip
---
pkg/yqlib/decoder_xml.go | 14 +++++--
pkg/yqlib/doc/usage/xml.md | 69 ++++++++++++++++++++++++++++++
pkg/yqlib/xml_test.go | 86 +++++++++++++++++++++++++++++++-------
3 files changed, 152 insertions(+), 17 deletions(-)
diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go
index 4a2a0c82b0..a3dffeb7f4 100644
--- a/pkg/yqlib/decoder_xml.go
+++ b/pkg/yqlib/decoder_xml.go
@@ -22,9 +22,11 @@ type xmlDecoder struct {
keepNamespace bool
useRawToken bool
finished bool
+ skipDirectives bool
+ skipProcInst bool
}
-func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool, keepNamespace bool, useRawToken bool) Decoder {
+func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool, keepNamespace bool, useRawToken bool, skipDirectives bool, skipProcInst bool) Decoder {
if contentName == "" {
contentName = "content"
}
@@ -37,6 +39,8 @@ func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool,
useRawToken: useRawToken,
directiveName: "_directive_",
procInstPrefix: "_procInst_",
+ skipDirectives: skipDirectives,
+ skipProcInst: skipProcInst,
}
}
@@ -285,9 +289,13 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
}
case xml.ProcInst:
- elem.n.AddChild(dec.procInstPrefix+se.Target, &xmlNode{Data: string(se.Inst)})
+ if !dec.skipProcInst {
+ elem.n.AddChild(dec.procInstPrefix+se.Target, &xmlNode{Data: string(se.Inst)})
+ }
case xml.Directive:
- elem.n.AddChild(dec.directiveName, &xmlNode{Data: string(se)})
+ if !dec.skipDirectives {
+ elem.n.AddChild(dec.directiveName, &xmlNode{Data: string(se)})
+ }
}
}
diff --git a/pkg/yqlib/doc/usage/xml.md b/pkg/yqlib/doc/usage/xml.md
index 729c8dc104..3d668506e5 100644
--- a/pkg/yqlib/doc/usage/xml.md
+++ b/pkg/yqlib/doc/usage/xml.md
@@ -30,6 +30,7 @@ yq -p=xml '.' sample.xml
```
will output
```yaml
+_procInst_xml: version="1.0" encoding="UTF-8"
cat:
says: meow
legs: "4"
@@ -54,6 +55,7 @@ yq -p=xml ' (.. | select(tag == "!!str")) |= from_yaml' sample.xml
```
will output
```yaml
+_procInst_xml: version="1.0" encoding="UTF-8"
cat:
says: meow
legs: 4
@@ -75,6 +77,7 @@ yq -p=xml '.' sample.xml
```
will output
```yaml
+_procInst_xml: version="1.0" encoding="UTF-8"
animal:
- cat
- goat
@@ -96,6 +99,7 @@ yq -p=xml '.' sample.xml
```
will output
```yaml
+_procInst_xml: version="1.0" encoding="UTF-8"
cat:
+legs: "4"
legs: "7"
@@ -115,6 +119,7 @@ yq -p=xml '.' sample.xml
```
will output
```yaml
+_procInst_xml: version="1.0" encoding="UTF-8"
cat:
+content: meow
+legs: "4"
@@ -141,6 +146,12 @@ yq -p=xml '.' sample.xml
```
will output
```yaml
+_procInst_xml: version="1.0"
+_directive_: |-
+ DOCTYPE root [
+ <!ENTITY writer "Blah.">
+ <!ENTITY copyright "Blah">
+ ]
root:
item: '&writer;©right;'
```
@@ -207,11 +218,13 @@ yq -p=xml -o=xml --xml-keep-namespace '.' sample.xml
```
will output
```xml
+<?xml version="1.0"?>
<map xmlns="some-namespace" xmlns:xsi="some-instance" some-instance:schemaLocation="some-url"></map>
```
instead of
```xml
+<?xml version="1.0"?>
<map xmlns="some-namespace" xsi="some-instance" schemaLocation="some-url"></map>
```
@@ -230,11 +243,13 @@ yq -p=xml -o=xml --xml-keep-namespace --xml-raw-token '.' sample.xml
```
will output
```xml
+<?xml version="1.0"?>
<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
```
instead of
```xml
+<?xml version="1.0"?>
<map xmlns="some-namespace" xsi="some-instance" schemaLocation="some-url"></map>
```
@@ -339,6 +354,32 @@ will output
</cat><!-- below_cat -->
```
+## Encode: doctype and xml declaration
+Use the special xml names to add/modify proc instructions and directives.
+
+Given a sample.yml file of:
+```yaml
+_procInst_xml: version="1.0"
+_directive_: 'DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" '
+apple:
+ _procInst_coolioo: version="1.0"
+ _directive_: 'CATYPE meow purr puss '
+ b: things
+
+```
+then
+```bash
+yq -o=xml '.' sample.yml
+```
+will output
+```xml
+<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<apple><?coolioo version="1.0"?><!CATYPE meow purr puss >
+ <b>things</b>
+</apple>
+```
+
## Round trip: with comments
A best effort is made, but comment positions and white space are not preserved perfectly.
@@ -380,3 +421,31 @@ in d before -->
</cat><!-- after cat -->
```
+## Roundtrip: with doctype and declaration
+yq parses XML proc instructions and directives into nodes.
+Unfortunately the underlying XML parser loses whitespace information.
+
+Given a sample.xml file of:
+```xml
+<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<apple>
+ <?coolioo version="1.0"?>
+ <!CATYPE meow purr puss >
+ <b>things</b>
+</apple>
+
+```
+then
+```bash
+yq -p=xml -o=xml '.' sample.xml
+```
+will output
+```xml
+<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<apple><?coolioo version="1.0"?><!CATYPE meow purr puss >
+ <b>things</b>
+</apple>
+```
+
diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go
index caaf52deba..34b85b15f4 100644
--- a/pkg/yqlib/xml_test.go
+++ b/pkg/yqlib/xml_test.go
@@ -159,13 +159,15 @@ const inputXMLWithNamespacedAttr = `
</map>
`
-const expectedYAMLWithNamespacedAttr = `map:
+const expectedYAMLWithNamespacedAttr = `_procInst_xml: version="1.0"
+map:
+xmlns: some-namespace
+xmlns:xsi: some-instance
+some-instance:schemaLocation: some-url
`
-const expectedYAMLWithRawNamespacedAttr = `map:
+const expectedYAMLWithRawNamespacedAttr = `_procInst_xml: version="1.0"
+map:
+xmlns: some-namespace
+xmlns:xsi: some-instance
+xsi:schemaLocation: some-url
@@ -181,10 +183,44 @@ const xmlWithCustomDtd = `
<item>&writer;©right;</item>
</root>`
-const expectedDtd = `root:
+const expectedDtd = `_procInst_xml: version="1.0"
+_directive_: |-
+ DOCTYPE root [
+ <!ENTITY writer "Blah.">
+ <!ENTITY copyright "Blah">
+ ]
+root:
item: '&writer;©right;'
`
+const expectedSkippedDtd = `root:
+ item: '&writer;©right;'
+`
+
+const xmlWithProcInstAndDirectives = `<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<apple>
+ <?coolioo version="1.0"?>
+ <!CATYPE meow purr puss >
+ <b>things</b>
+</apple>
+`
+
+const yamlWithProcInstAndDirectives = `_procInst_xml: version="1.0"
+_directive_: 'DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" '
+apple:
+ _procInst_coolioo: version="1.0"
+ _directive_: 'CATYPE meow purr puss '
+ b: things
+`
+
+const expectedXmlWithProcInstAndDirectives = `<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<apple><?coolioo version="1.0"?><!CATYPE meow purr puss >
+ <b>things</b>
+</apple>
+`
+
var xmlScenarios = []formatScenario{
{
description: "Parse xml: simple",
@@ -219,10 +255,17 @@ var xmlScenarios = []formatScenario{
},
{
description: "Parse xml: custom dtd",
- subdescription: "DTD entities are ignored.",
+ subdescription: "DTD entities are processed as directives.",
input: xmlWithCustomDtd,
expected: expectedDtd,
},
+ {
+ description: "Parse xml: custom dtd",
+ subdescription: "DTD entities are processed as directives.",
+ input: xmlWithCustomDtd,
+ expected: expectedSkippedDtd,
+ scenarioType: "c",
+ },
{
description: "Parse xml: with comments",
subdescription: "A best attempt is made to preserve comments.",
@@ -341,6 +384,13 @@ var xmlScenarios = []formatScenario{
expected: expectedXMLWithComments,
scenarioType: "encode",
},
+ {
+ description: "Encode: doctype and xml declaration",
+ subdescription: "Use the special xml names to add/modify proc instructions and directives.",
+ input: yamlWithProcInstAndDirectives,
+ expected: expectedXmlWithProcInstAndDirectives,
+ scenarioType: "encode",
+ },
{
description: "Round trip: with comments",
subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.",
@@ -348,21 +398,29 @@ var xmlScenarios = []formatScenario{
expected: expectedRoundtripXMLWithComments,
scenarioType: "roundtrip",
},
+ {
+ description: "Roundtrip: with doctype and declaration",
+ subdescription: "yq parses XML proc instructions and directives into nodes.\nUnfortunately the underlying XML parser loses whitespace information.",
+ input: xmlWithProcInstAndDirectives,
+ expected: expectedXmlWithProcInstAndDirectives,
+ scenarioType: "roundtrip",
+ },
}
func testXMLScenario(t *testing.T, s formatScenario) {
switch s.scenarioType {
case "", "decode":
- test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewYamlEncoder(4, false, true, true)), s.description)
+ test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false, false, false), NewYamlEncoder(4, false, true, true)), s.description)
case "encode":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, "+", "+content")), s.description)
case "roundtrip":
- test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewXMLEncoder(2, "+", "+content")), s.description)
+ test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false, false, false), NewXMLEncoder(2, "+", "+content")), s.description)
case "decode-keep-ns":
- test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, false), NewYamlEncoder(2, false, true, true)), s.description)
+ test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, false, false, false), NewYamlEncoder(2, false, true, true)), s.description)
case "decode-raw-token":
- test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, true), NewYamlEncoder(2, false, true, true)), s.description)
-
+ test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, true, false, false), NewYamlEncoder(2, false, true, true)), s.description)
+ case "encode-":
+ test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, true, true, true), NewYamlEncoder(2, false, true, true)), s.description)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
@@ -428,10 +486,10 @@ func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "```bash\nyq -p=xml -o=xml --xml-keep-namespace '.' sample.xml\n```\n")
writeOrPanic(w, "will output\n")
- writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, false), NewXMLEncoder(2, "+", "+content"))))
+ writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, false, false, false), NewXMLEncoder(2, "+", "+content"))))
writeOrPanic(w, "instead of\n")
- writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewXMLEncoder(2, "+", "+content"))))
+ writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false, false, false), NewXMLEncoder(2, "+", "+content"))))
}
func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario) {
@@ -449,10 +507,10 @@ func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario)
writeOrPanic(w, "```bash\nyq -p=xml -o=xml --xml-keep-namespace --xml-raw-token '.' sample.xml\n```\n")
writeOrPanic(w, "will output\n")
- writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, true), NewXMLEncoder(2, "+", "+content"))))
+ writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, true, false, false), NewXMLEncoder(2, "+", "+content"))))
writeOrPanic(w, "instead of\n")
- writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewXMLEncoder(2, "+", "+content"))))
+ writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false, false, false), NewXMLEncoder(2, "+", "+content"))))
}
func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
@@ -488,7 +546,7 @@ func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "```bash\nyq -p=xml -o=xml '.' sample.xml\n```\n")
writeOrPanic(w, "will output\n")
- writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewXMLEncoder(2, "+", "+content"))))
+ writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false, false, false), NewXMLEncoder(2, "+", "+content"))))
}
func TestXMLScenarios(t *testing.T) {
From 585cafc1c05ce5f8396fb9256d5cb08421d1d58c Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Fri, 21 Oct 2022 15:58:47 +1100
Subject: [PATCH 04/10] wip
---
pkg/yqlib/decoder_xml.go | 51 +++-------
pkg/yqlib/doc/usage/xml.md | 79 ++++++++++-----
pkg/yqlib/encoder_xml.go | 52 +++++-----
pkg/yqlib/lexer_participle.go | 2 +-
pkg/yqlib/lib.go | 20 ++++
pkg/yqlib/operator_encoder_decoder.go | 9 +-
pkg/yqlib/xml_test.go | 134 +++++++++++++++++---------
7 files changed, 206 insertions(+), 141 deletions(-)
diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go
index a3dffeb7f4..d74c579a65 100644
--- a/pkg/yqlib/decoder_xml.go
+++ b/pkg/yqlib/decoder_xml.go
@@ -12,35 +12,16 @@ import (
)
type xmlDecoder struct {
- reader io.Reader
- readAnything bool
- attributePrefix string
- directiveName string
- procInstPrefix string
- contentName string
- strictMode bool
- keepNamespace bool
- useRawToken bool
- finished bool
- skipDirectives bool
- skipProcInst bool
+ reader io.Reader
+ readAnything bool
+ finished bool
+ prefs xmlPreferences
}
-func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool, keepNamespace bool, useRawToken bool, skipDirectives bool, skipProcInst bool) Decoder {
- if contentName == "" {
- contentName = "content"
- }
+func NewXMLDecoder(prefs xmlPreferences) Decoder {
return &xmlDecoder{
- attributePrefix: attributePrefix,
- contentName: contentName,
- finished: false,
- strictMode: strictMode,
- keepNamespace: keepNamespace,
- useRawToken: useRawToken,
- directiveName: "_directive_",
- procInstPrefix: "_procInst_",
- skipDirectives: skipDirectives,
- skipProcInst: skipProcInst,
+ finished: false,
+ prefs: prefs,
}
}
@@ -75,7 +56,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
yamlNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
if len(n.Data) > 0 {
- label := dec.contentName
+ label := dec.prefs.ContentName
labelNode := createScalarNode(label, label)
labelNode.HeadComment = dec.processComment(n.HeadComment)
labelNode.FootComment = dec.processComment(n.FootComment)
@@ -211,7 +192,7 @@ type element struct {
// of the map keys.
func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
xmlDec := xml.NewDecoder(dec.reader)
- xmlDec.Strict = dec.strictMode
+ xmlDec.Strict = dec.prefs.StrictMode
// That will convert the charset if the provided XML is non-UTF-8
xmlDec.CharsetReader = charset.NewReaderLabel
@@ -222,7 +203,7 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
}
getToken := func() (xml.Token, error) {
- if dec.useRawToken {
+ if dec.prefs.UseRawToken {
return xmlDec.RawToken()
}
return xmlDec.Token()
@@ -250,12 +231,12 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
// Extract attributes as children
for _, a := range se.Attr {
- if dec.keepNamespace {
+ if dec.prefs.KeepNamespace {
if a.Name.Space != "" {
a.Name.Local = a.Name.Space + ":" + a.Name.Local
}
}
- elem.n.AddChild(dec.attributePrefix+a.Name.Local, &xmlNode{Data: a.Value})
+ elem.n.AddChild(dec.prefs.AttributePrefix+a.Name.Local, &xmlNode{Data: a.Value})
}
case xml.CharData:
// Extract XML data (if any)
@@ -289,12 +270,12 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
}
case xml.ProcInst:
- if !dec.skipProcInst {
- elem.n.AddChild(dec.procInstPrefix+se.Target, &xmlNode{Data: string(se.Inst)})
+ if !dec.prefs.SkipProcInst {
+ elem.n.AddChild(dec.prefs.ProcInstPrefix+se.Target, &xmlNode{Data: string(se.Inst)})
}
case xml.Directive:
- if !dec.skipDirectives {
- elem.n.AddChild(dec.directiveName, &xmlNode{Data: string(se)})
+ if !dec.prefs.SkipDirectives {
+ elem.n.AddChild(dec.prefs.DirectiveName, &xmlNode{Data: string(se)})
}
}
}
diff --git a/pkg/yqlib/doc/usage/xml.md b/pkg/yqlib/doc/usage/xml.md
index 3d668506e5..f0d80bf48b 100644
--- a/pkg/yqlib/doc/usage/xml.md
+++ b/pkg/yqlib/doc/usage/xml.md
@@ -30,7 +30,7 @@ yq -p=xml '.' sample.xml
```
will output
```yaml
-_procInst_xml: version="1.0" encoding="UTF-8"
++p_xml: version="1.0" encoding="UTF-8"
cat:
says: meow
legs: "4"
@@ -55,7 +55,7 @@ yq -p=xml ' (.. | select(tag == "!!str")) |= from_yaml' sample.xml
```
will output
```yaml
-_procInst_xml: version="1.0" encoding="UTF-8"
++p_xml: version="1.0" encoding="UTF-8"
cat:
says: meow
legs: 4
@@ -77,7 +77,7 @@ yq -p=xml '.' sample.xml
```
will output
```yaml
-_procInst_xml: version="1.0" encoding="UTF-8"
++p_xml: version="1.0" encoding="UTF-8"
animal:
- cat
- goat
@@ -99,9 +99,9 @@ yq -p=xml '.' sample.xml
```
will output
```yaml
-_procInst_xml: version="1.0" encoding="UTF-8"
++p_xml: version="1.0" encoding="UTF-8"
cat:
- +legs: "4"
+ +@legs: "4"
legs: "7"
```
@@ -119,14 +119,14 @@ yq -p=xml '.' sample.xml
```
will output
```yaml
-_procInst_xml: version="1.0" encoding="UTF-8"
++p_xml: version="1.0" encoding="UTF-8"
cat:
+content: meow
- +legs: "4"
+ +@legs: "4"
```
## Parse xml: custom dtd
-DTD entities are ignored.
+DTD entities are processed as directives.
Given a sample.xml file of:
```xml
@@ -142,18 +142,45 @@ Given a sample.xml file of:
```
then
```bash
-yq -p=xml '.' sample.xml
+yq -p=xml -o=xml '.' sample.xml
```
will output
-```yaml
-_procInst_xml: version="1.0"
-_directive_: |-
- DOCTYPE root [
- <!ENTITY writer "Blah.">
- <!ENTITY copyright "Blah">
- ]
-root:
- item: '&writer;©right;'
+```xml
+<?xml version="1.0"?>
+<!DOCTYPE root [
+<!ENTITY writer "Blah.">
+<!ENTITY copyright "Blah">
+]>
+<root>
+ <item>&writer;&copyright;</item>
+</root>
+```
+
+## Parse xml: skip custom dtd
+DTDs are directives, skip over directives to skip DTDs.
+
+Given a sample.xml file of:
+```xml
+
+<?xml version="1.0"?>
+<!DOCTYPE root [
+<!ENTITY writer "Blah.">
+<!ENTITY copyright "Blah">
+]>
+<root>
+ <item>&writer;©right;</item>
+</root>
+```
+then
+```bash
+yq -p=xml -o=xml --xml-skip-directives '.' sample.xml
+```
+will output
+```xml
+<?xml version="1.0"?>
+<root>
+ <item>&writer;&copyright;</item>
+</root>
```
## Parse xml: with comments
@@ -225,7 +252,7 @@ will output
instead of
```xml
<?xml version="1.0"?>
-<map xmlns="some-namespace" xsi="some-instance" schemaLocation="some-url"></map>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" some-instance:schemaLocation="some-url"></map>
```
## Parse xml: keep raw attribute namespace
@@ -244,7 +271,7 @@ yq -p=xml -o=xml --xml-keep-namespace --xml-raw-token '.' sample.xml
will output
```xml
<?xml version="1.0"?>
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" some-instance:schemaLocation="some-url"></map>
```
instead of
@@ -293,7 +320,7 @@ Fields with the matching xml-attribute-prefix are assumed to be attributes.
Given a sample.yml file of:
```yaml
cat:
- +name: tiger
+ +@name: tiger
meows: true
```
@@ -314,7 +341,7 @@ Fields with the matching xml-content-name is assumed to be content.
Given a sample.yml file of:
```yaml
cat:
- +name: tiger
+ +@name: tiger
+content: cool
```
@@ -359,11 +386,11 @@ Use the special xml names to add/modify proc instructions and directives.
Given a sample.yml file of:
```yaml
-_procInst_xml: version="1.0"
-_directive_: 'DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" '
++p_xml: version="1.0"
++directive: 'DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" '
apple:
- _procInst_coolioo: version="1.0"
- _directive_: 'CATYPE meow purr puss '
+ +p_coolioo: version="1.0"
+ +directive: 'CATYPE meow purr puss '
b: things
```
diff --git a/pkg/yqlib/encoder_xml.go b/pkg/yqlib/encoder_xml.go
index ecb6a431ae..4e3dd15171 100644
--- a/pkg/yqlib/encoder_xml.go
+++ b/pkg/yqlib/encoder_xml.go
@@ -9,24 +9,19 @@ import (
yaml "gopkg.in/yaml.v3"
)
-var XMLPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content", StrictMode: false, UseRawToken: false}
-
type xmlEncoder struct {
- attributePrefix string
- contentName string
- indentString string
- directiveName string
- procInstPrefix string
- writer io.Writer
+ indentString string
+ writer io.Writer
+ prefs xmlPreferences
}
-func NewXMLEncoder(indent int, attributePrefix string, contentName string) Encoder {
+func NewXMLEncoder(indent int, prefs xmlPreferences) Encoder {
var indentString = ""
for index := 0; index < indent; index++ {
indentString = indentString + " "
}
- return &xmlEncoder{attributePrefix, contentName, indentString, "_directive_", "_procInst_", nil}
+ return &xmlEncoder{indentString, nil, prefs}
}
func (e *xmlEncoder) CanHandleAliases() bool {
@@ -97,8 +92,8 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) er
return err
}
- if strings.HasPrefix(key.Value, e.procInstPrefix) {
- name := strings.Replace(key.Value, e.procInstPrefix, "", 1)
+ if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) {
+ name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
if err := encoder.EncodeToken(procInst); err != nil {
return err
@@ -106,7 +101,7 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) er
if _, err := e.writer.Write([]byte("\n")); err != nil {
log.Warning("Unable to write newline, skipping: %w", err)
}
- } else if key.Value == e.directiveName {
+ } else if key.Value == e.prefs.DirectiveName {
var directive xml.Directive = []byte(value.Value)
if err := encoder.EncodeToken(directive); err != nil {
return err
@@ -205,6 +200,13 @@ func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yaml.Node, start xm
return e.encodeComment(encoder, footComment(node))
}
+func (e *xmlEncoder) isAttribute(name string) bool {
+ return strings.HasPrefix(name, e.prefs.AttributePrefix) &&
+ name != e.prefs.ContentName &&
+ name != e.prefs.DirectiveName &&
+ !strings.HasPrefix(name, e.prefs.ProcInstPrefix)
+}
+
func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error {
log.Debug("its a map")
@@ -213,9 +215,9 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.
key := node.Content[i]
value := node.Content[i+1]
- if strings.HasPrefix(key.Value, e.attributePrefix) && key.Value != e.contentName {
+ if e.isAttribute(key.Value) {
if value.Kind == yaml.ScalarNode {
- attributeName := strings.Replace(key.Value, e.attributePrefix, "", 1)
+ attributeName := strings.Replace(key.Value, e.prefs.AttributePrefix, "", 1)
start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: attributeName}, Value: value.Value})
} else {
return fmt.Errorf("cannot use %v as attribute, only scalars are supported", value.Tag)
@@ -237,24 +239,18 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.
if err != nil {
return err
}
- if strings.HasPrefix(key.Value, e.procInstPrefix) {
- name := strings.Replace(key.Value, e.procInstPrefix, "", 1)
+ if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) {
+ name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
if err := encoder.EncodeToken(procInst); err != nil {
return err
}
- } else if key.Value == e.directiveName {
+ } else if key.Value == e.prefs.DirectiveName {
var directive xml.Directive = []byte(value.Value)
if err := encoder.EncodeToken(directive); err != nil {
return err
}
- } else if !strings.HasPrefix(key.Value, e.attributePrefix) && key.Value != e.contentName {
- start := xml.StartElement{Name: xml.Name{Local: key.Value}}
- err := e.doEncode(encoder, value, start)
- if err != nil {
- return err
- }
- } else if key.Value == e.contentName {
+ } else if key.Value == e.prefs.ContentName {
// directly encode the contents
err = e.encodeComment(encoder, headAndLineComment(value))
if err != nil {
@@ -269,6 +265,12 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.
if err != nil {
return err
}
+ } else if !e.isAttribute(key.Value) {
+ start := xml.StartElement{Name: xml.Name{Local: key.Value}}
+ err := e.doEncode(encoder, value, start)
+ if err != nil {
+ return err
+ }
}
err = e.encodeComment(encoder, footComment(key))
if err != nil {
diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go
index 170424a760..6cc745ccea 100644
--- a/pkg/yqlib/lexer_participle.go
+++ b/pkg/yqlib/lexer_participle.go
@@ -76,7 +76,7 @@ var participleYqRules = []*participleYqRule{
{"Base64d", `@base64d`, decodeOp(Base64InputFormat), 0},
{"Base64", `@base64`, encodeWithIndent(Base64OutputFormat, 0), 0},
- {"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode, XMLPreferences.KeepNamespace, XMLPreferences.UseRawToken), false), 0},
+ {"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(XMLPreferences), false), 0},
{"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder(), false), 0},
diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go
index e5f42a49ab..18383383cf 100644
--- a/pkg/yqlib/lib.go
+++ b/pkg/yqlib/lib.go
@@ -27,8 +27,28 @@ type xmlPreferences struct {
StrictMode bool
KeepNamespace bool
UseRawToken bool
+ ProcInstPrefix string
+ DirectiveName string
+ SkipProcInst bool
+ SkipDirectives bool
}
+func NewDefaultXmlPreferences() xmlPreferences {
+ return xmlPreferences{
+ AttributePrefix: "+@",
+ ContentName: "+content",
+ StrictMode: false,
+ KeepNamespace: true,
+ UseRawToken: false,
+ ProcInstPrefix: "+p_",
+ DirectiveName: "+directive",
+ SkipProcInst: false,
+ SkipDirectives: false,
+ }
+}
+
+var XMLPreferences = NewDefaultXmlPreferences()
+
var log = logging.MustGetLogger("yq-lib")
var PrettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not)) ) style=""`
diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go
index afb321ef9a..e8b8a963ef 100644
--- a/pkg/yqlib/operator_encoder_decoder.go
+++ b/pkg/yqlib/operator_encoder_decoder.go
@@ -23,7 +23,7 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
case YamlOutputFormat:
return NewYamlEncoder(indent, false, true, true)
case XMLOutputFormat:
- return NewXMLEncoder(indent, XMLPreferences.AttributePrefix, XMLPreferences.ContentName)
+ return NewXMLEncoder(indent, XMLPreferences)
case Base64OutputFormat:
return NewBase64Encoder()
}
@@ -104,12 +104,7 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
case YamlInputFormat:
decoder = NewYamlDecoder()
case XMLInputFormat:
- decoder = NewXMLDecoder(
- XMLPreferences.AttributePrefix,
- XMLPreferences.ContentName,
- XMLPreferences.StrictMode,
- XMLPreferences.KeepNamespace,
- XMLPreferences.UseRawToken)
+ decoder = NewXMLDecoder(XMLPreferences)
case Base64InputFormat:
decoder = NewBase64Decoder()
case PropertiesInputFormat:
diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go
index 34b85b15f4..0c69a24eca 100644
--- a/pkg/yqlib/xml_test.go
+++ b/pkg/yqlib/xml_test.go
@@ -58,7 +58,7 @@ cat:
d:
# in d before
z:
- +sweet: cool
+ +@sweet: cool
# in d after
# in y after
# in_cat_after
@@ -98,11 +98,11 @@ cat:
d:
- # in d before
z:
- +sweet: cool
+ +@sweet: cool
# in d after
- # in d2 before
z:
- +sweet: cool2
+ +@sweet: cool2
# in d2 after
# in y after
# in_cat_after
@@ -159,18 +159,18 @@ const inputXMLWithNamespacedAttr = `
</map>
`
-const expectedYAMLWithNamespacedAttr = `_procInst_xml: version="1.0"
+const expectedYAMLWithNamespacedAttr = `+p_xml: version="1.0"
map:
- +xmlns: some-namespace
- +xmlns:xsi: some-instance
- +some-instance:schemaLocation: some-url
+ +@xmlns: some-namespace
+ +@xmlns:xsi: some-instance
+ +@some-instance:schemaLocation: some-url
`
-const expectedYAMLWithRawNamespacedAttr = `_procInst_xml: version="1.0"
+const expectedYAMLWithRawNamespacedAttr = `+p_xml: version="1.0"
map:
- +xmlns: some-namespace
- +xmlns:xsi: some-instance
- +xsi:schemaLocation: some-url
+ +@xmlns: some-namespace
+ +@xmlns:xsi: some-instance
+ +@xsi:schemaLocation: some-url
`
const xmlWithCustomDtd = `
@@ -183,18 +183,19 @@ const xmlWithCustomDtd = `
<item>&writer;©right;</item>
</root>`
-const expectedDtd = `_procInst_xml: version="1.0"
-_directive_: |-
- DOCTYPE root [
- <!ENTITY writer "Blah.">
- <!ENTITY copyright "Blah">
- ]
-root:
- item: '&writer;©right;'
+const expectedDtd = `<?xml version="1.0"?>
+<!DOCTYPE root [
+<!ENTITY writer "Blah.">
+<!ENTITY copyright "Blah">
+]>
+<root>
+ <item>&writer;&copyright;</item>
+</root>
`
-const expectedSkippedDtd = `root:
- item: '&writer;©right;'
+const expectedSkippedDtd = `<root>
+ <item>&writer;©right;</item>
+</root>
`
const xmlWithProcInstAndDirectives = `<?xml version="1.0"?>
@@ -206,11 +207,11 @@ const xmlWithProcInstAndDirectives = `<?xml version="1.0"?>
</apple>
`
-const yamlWithProcInstAndDirectives = `_procInst_xml: version="1.0"
-_directive_: 'DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" '
+const yamlWithProcInstAndDirectives = `+p_xml: version="1.0"
++directive: 'DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" '
apple:
- _procInst_coolioo: version="1.0"
- _directive_: 'CATYPE meow purr puss '
+ +p_coolioo: version="1.0"
+ +directive: 'CATYPE meow purr puss '
b: things
`
@@ -258,13 +259,14 @@ var xmlScenarios = []formatScenario{
subdescription: "DTD entities are processed as directives.",
input: xmlWithCustomDtd,
expected: expectedDtd,
+ scenarioType: "roundtrip",
},
{
- description: "Parse xml: custom dtd",
- subdescription: "DTD entities are processed as directives.",
+ description: "Parse xml: skip custom dtd",
+ subdescription: "DTDs are directives, skip over directives to skip DTDs.",
input: xmlWithCustomDtd,
expected: expectedSkippedDtd,
- scenarioType: "c",
+ scenarioType: "roundtrip-skip-directives",
},
{
description: "Parse xml: with comments",
@@ -360,20 +362,21 @@ var xmlScenarios = []formatScenario{
{
description: "Encode xml: attributes",
subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
- input: "cat:\n +name: tiger\n meows: true\n",
+ input: "cat:\n +@name: tiger\n meows: true\n",
expected: "<cat name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
scenarioType: "encode",
},
{
+ description: "double prefix",
skipDoc: true,
- input: "cat:\n ++name: tiger\n meows: true\n",
- expected: "<cat +name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
+ input: "cat:\n +@+@name: tiger\n meows: true\n",
+ expected: "<cat +@name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
scenarioType: "encode",
},
{
description: "Encode xml: attributes with content",
subdescription: "Fields with the matching xml-content-name is assumed to be content.",
- input: "cat:\n +name: tiger\n +content: cool\n",
+ input: "cat:\n +@name: tiger\n +content: cool\n",
expected: "<cat name=\"tiger\">cool</cat>\n",
scenarioType: "encode",
},
@@ -410,17 +413,23 @@ var xmlScenarios = []formatScenario{
func testXMLScenario(t *testing.T, s formatScenario) {
switch s.scenarioType {
case "", "decode":
- test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false, false, false), NewYamlEncoder(4, false, true, true)), s.description)
+ test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(XMLPreferences), NewYamlEncoder(4, false, true, true)), s.description)
case "encode":
- test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, "+", "+content")), s.description)
+ test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, XMLPreferences)), s.description)
case "roundtrip":
- test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false, false, false), NewXMLEncoder(2, "+", "+content")), s.description)
+ test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(XMLPreferences), NewXMLEncoder(2, XMLPreferences)), s.description)
case "decode-keep-ns":
- test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, false, false, false), NewYamlEncoder(2, false, true, true)), s.description)
+ prefs := NewDefaultXmlPreferences()
+ prefs.KeepNamespace = true
+ test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, true, true)), s.description)
case "decode-raw-token":
- test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, true, false, false), NewYamlEncoder(2, false, true, true)), s.description)
- case "encode-":
- test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, true, true, true), NewYamlEncoder(2, false, true, true)), s.description)
+ prefs := NewDefaultXmlPreferences()
+ prefs.UseRawToken = true
+ test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, true, true)), s.description)
+ case "roundtrip-skip-directives":
+ prefs := NewDefaultXmlPreferences()
+ prefs.SkipDirectives = true
+ test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs)), s.description)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
@@ -443,6 +452,8 @@ func documentXMLScenario(t *testing.T, w *bufio.Writer, i interface{}) {
documentXMLDecodeKeepNsScenario(w, s)
case "decode-raw-token":
documentXMLDecodeKeepNsRawTokenScenario(w, s)
+ case "roundtrip-skip-directives":
+ documentXMLSkipDirectrivesScenario(w, s)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
@@ -468,7 +479,7 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=xml '%v' sample.xml\n```\n", expression))
writeOrPanic(w, "will output\n")
- writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false), NewYamlEncoder(2, false, true, true))))
+ writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(XMLPreferences), NewYamlEncoder(2, false, true, true))))
}
func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
@@ -485,11 +496,14 @@ func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "then\n")
writeOrPanic(w, "```bash\nyq -p=xml -o=xml --xml-keep-namespace '.' sample.xml\n```\n")
writeOrPanic(w, "will output\n")
+ prefs := NewDefaultXmlPreferences()
+ prefs.KeepNamespace = true
+ writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
- writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, false, false, false), NewXMLEncoder(2, "+", "+content"))))
-
+ prefsWithout := NewDefaultXmlPreferences()
+ prefs.KeepNamespace = false
writeOrPanic(w, "instead of\n")
- writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false, false, false), NewXMLEncoder(2, "+", "+content"))))
+ writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
}
func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario) {
@@ -507,10 +521,16 @@ func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario)
writeOrPanic(w, "```bash\nyq -p=xml -o=xml --xml-keep-namespace --xml-raw-token '.' sample.xml\n```\n")
writeOrPanic(w, "will output\n")
- writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, true, true, false, false), NewXMLEncoder(2, "+", "+content"))))
+ prefs := NewDefaultXmlPreferences()
+ prefs.KeepNamespace = true
+
+ writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
+
+ prefsWithout := NewDefaultXmlPreferences()
+ prefsWithout.KeepNamespace = false
writeOrPanic(w, "instead of\n")
- writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false, false, false), NewXMLEncoder(2, "+", "+content"))))
+ writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout))))
}
func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
@@ -528,7 +548,7 @@ func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "```bash\nyq -o=xml '.' sample.yml\n```\n")
writeOrPanic(w, "will output\n")
- writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, "+", "+content"))))
+ writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, XMLPreferences))))
}
func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
@@ -546,7 +566,27 @@ func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "```bash\nyq -p=xml -o=xml '.' sample.xml\n```\n")
writeOrPanic(w, "will output\n")
- writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false, false, false, false, false), NewXMLEncoder(2, "+", "+content"))))
+ writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(XMLPreferences), NewXMLEncoder(2, XMLPreferences))))
+}
+
+func documentXMLSkipDirectrivesScenario(w *bufio.Writer, s formatScenario) {
+ writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+ if s.subdescription != "" {
+ writeOrPanic(w, s.subdescription)
+ writeOrPanic(w, "\n\n")
+ }
+
+ writeOrPanic(w, "Given a sample.xml file of:\n")
+ writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
+
+ writeOrPanic(w, "then\n")
+ writeOrPanic(w, "```bash\nyq -p=xml -o=xml --xml-skip-directives '.' sample.xml\n```\n")
+ writeOrPanic(w, "will output\n")
+ prefs := NewDefaultXmlPreferences()
+ prefs.SkipDirectives = true
+
+ writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs))))
}
func TestXMLScenarios(t *testing.T) {
From 9546eb893924ba8fd43e76c879f40ac8168ee5e2 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Fri, 21 Oct 2022 15:59:13 +1100
Subject: [PATCH 05/10] wip
---
cmd/constant.go | 6 ------
cmd/root.go | 17 +++++++++--------
cmd/utils.go | 4 ++--
3 files changed, 11 insertions(+), 16 deletions(-)
diff --git a/cmd/constant.go b/cmd/constant.go
index a152e12aae..ef0c31dbd8 100644
--- a/cmd/constant.go
+++ b/cmd/constant.go
@@ -8,12 +8,6 @@ var outputToJSON = false
var outputFormat = "yaml"
var inputFormat = "yaml"
-var xmlAttributePrefix = "+"
-var xmlContentName = "+content"
-var xmlStrictMode = false
-var xmlKeepNamespace = true
-var xmlUseRawToken = true
-
var exitStatus = false
var forceColor = false
var forceNoColor = false
diff --git a/cmd/root.go b/cmd/root.go
index 3951ed06bc..2f802b33ac 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -52,9 +52,6 @@ yq -P sample.json
logging.SetBackend(backend)
yqlib.InitExpressionParser()
- yqlib.XMLPreferences.AttributePrefix = xmlAttributePrefix
- yqlib.XMLPreferences.ContentName = xmlContentName
- yqlib.XMLPreferences.StrictMode = xmlStrictMode
},
}
@@ -69,11 +66,15 @@ yq -P sample.json
rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
- rootCmd.PersistentFlags().StringVar(&xmlAttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
- rootCmd.PersistentFlags().StringVar(&xmlContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")
- rootCmd.PersistentFlags().BoolVar(&xmlStrictMode, "xml-strict-mode", false, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
- rootCmd.PersistentFlags().BoolVar(&xmlKeepNamespace, "xml-keep-namespace", true, "enables keeping namespace after parsing attributes")
- rootCmd.PersistentFlags().BoolVar(&xmlUseRawToken, "xml-raw-token", true, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
+ rootCmd.PersistentFlags().StringVar(&yqlib.XMLPreferences.AttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
+ rootCmd.PersistentFlags().StringVar(&yqlib.XMLPreferences.ContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")
+ rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.StrictMode, "xml-strict-mode", false, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
+ rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.KeepNamespace, "xml-keep-namespace", true, "enables keeping namespace after parsing attributes")
+ rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.UseRawToken, "xml-raw-token", true, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
+ rootCmd.PersistentFlags().StringVar(&yqlib.XMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", "+p_", "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
+ rootCmd.PersistentFlags().StringVar(&yqlib.XMLPreferences.DirectiveName, "xml-directive-name", "+directive", "name for xml directives (e.g. <!DOCTYPE thing cat>)")
+ rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.SkipProcInst, "xml-skip-proc-inst", false, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
+ rootCmd.PersistentFlags().BoolVar(&yqlib.XMLPreferences.SkipDirectives, "xml-skip-directives", false, "skip over directives (e.g. <!DOCTYPE thing cat>)")
rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
diff --git a/cmd/utils.go b/cmd/utils.go
index e0aaf1d612..b201b1dbcd 100644
--- a/cmd/utils.go
+++ b/cmd/utils.go
@@ -63,7 +63,7 @@ func configureDecoder() (yqlib.Decoder, error) {
}
switch yqlibInputFormat {
case yqlib.XMLInputFormat:
- return yqlib.NewXMLDecoder(xmlAttributePrefix, xmlContentName, xmlStrictMode, xmlKeepNamespace, xmlUseRawToken), nil
+ return yqlib.NewXMLDecoder(yqlib.XMLPreferences), nil
case yqlib.PropertiesInputFormat:
return yqlib.NewPropertiesDecoder(), nil
case yqlib.JsonInputFormat:
@@ -107,7 +107,7 @@ func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
case yqlib.YamlOutputFormat:
return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar)
case yqlib.XMLOutputFormat:
- return yqlib.NewXMLEncoder(indent, xmlAttributePrefix, xmlContentName)
+ return yqlib.NewXMLEncoder(indent, yqlib.XMLPreferences)
}
panic("invalid encoder")
}
From 55383b9ce1bee9644d6b9da821867720d629b37b Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Fri, 21 Oct 2022 20:46:26 +1100
Subject: [PATCH 06/10] warn about attribute change
---
cmd/root.go | 9 ++++++++-
pkg/yqlib/encoder_xml.go | 21 ++++++++++++++++++++-
pkg/yqlib/lib.go | 2 +-
3 files changed, 29 insertions(+), 3 deletions(-)
diff --git a/cmd/root.go b/cmd/root.go
index 2f802b33ac..3a8a762158 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -47,11 +47,18 @@ yq -P sample.json
if verbose {
backend.SetLevel(logging.DEBUG, "")
} else {
- backend.SetLevel(logging.ERROR, "")
+ backend.SetLevel(logging.WARNING, "")
}
logging.SetBackend(backend)
yqlib.InitExpressionParser()
+ if (inputFormat == "x" || inputFormat == "xml") &&
+ outputFormat != "x" && outputFormat != "xml" &&
+ yqlib.XMLPreferences.AttributePrefix == "+" {
+ yqlib.GetLogger().Warning("The default xml-attribute-prefix will change in the next release to `+@` to avoid " +
+ "naming conflicts with the default content name, directive name and proc inst prefix. If you need to keep " +
+ "`+` please set that value explicityly with --xml-attribute-prefix.")
+ }
},
}
diff --git a/pkg/yqlib/encoder_xml.go b/pkg/yqlib/encoder_xml.go
index 4e3dd15171..012e64b75a 100644
--- a/pkg/yqlib/encoder_xml.go
+++ b/pkg/yqlib/encoder_xml.go
@@ -77,6 +77,23 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
}
func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) error {
+ // make sure <?xml .. ?> processing instructions are encoded first
+ for i := 0; i < len(node.Content); i += 2 {
+ key := node.Content[i]
+ value := node.Content[i+1]
+
+ if key.Value == (e.prefs.ProcInstPrefix + "xml") {
+ name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
+ procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
+ if err := encoder.EncodeToken(procInst); err != nil {
+ return err
+ }
+ if _, err := e.writer.Write([]byte("\n")); err != nil {
+ log.Warning("Unable to write newline, skipping: %w", err)
+ }
+ }
+ }
+
err := e.encodeComment(encoder, headAndLineComment(node))
if err != nil {
return err
@@ -92,7 +109,9 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) er
return err
}
- if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) {
+ if key.Value == (e.prefs.ProcInstPrefix + "xml") {
+ // dont double process these.
+ } else if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) {
name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
if err := encoder.EncodeToken(procInst); err != nil {
diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go
index 18383383cf..910ab80141 100644
--- a/pkg/yqlib/lib.go
+++ b/pkg/yqlib/lib.go
@@ -35,7 +35,7 @@ type xmlPreferences struct {
func NewDefaultXmlPreferences() xmlPreferences {
return xmlPreferences{
- AttributePrefix: "+@",
+ AttributePrefix: "+",
ContentName: "+content",
StrictMode: false,
KeepNamespace: true,
From cb86ec94ffefaa1dd9706d56280949edc0c85587 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sun, 23 Oct 2022 14:48:57 +1100
Subject: [PATCH 07/10] xml wip
---
pkg/yqlib/doc/usage/headers/xml.md | 27 ++++++++++++++++++++-
pkg/yqlib/doc/usage/xml.md | 35 +++++++++++++++++++++++----
pkg/yqlib/xml_test.go | 39 +++++++++++++++---------------
3 files changed, 76 insertions(+), 25 deletions(-)
diff --git a/pkg/yqlib/doc/usage/headers/xml.md b/pkg/yqlib/doc/usage/headers/xml.md
index fd8e79d67a..ce29869b21 100644
--- a/pkg/yqlib/doc/usage/headers/xml.md
+++ b/pkg/yqlib/doc/usage/headers/xml.md
@@ -4,4 +4,29 @@ Encode and decode to and from XML. Whitespace is not conserved for round trips -
Consecutive xml nodes with the same name are assumed to be arrays.
-XML content data and attributes are created as fields. This can be controlled by the `'--xml-attribute-prefix` and `--xml-content-name` flags - see below for examples.
+XML content data, attributes processing instructions and directives are all created as plain fields.
+
+This can be controlled by:
+
+| Flag | Default |Sample XML |
+| -- | -- | -- |
+ | `--xml-attribute-prefix` | `+` (changing to `+@` soon) | Legs in ```<cat legs="4"/>``` |
+ | `--xml-content-name` | `+content` | Meow in ```<cat>Meow <fur>true</true></cat>``` |
+ | `--xml-directive-name` | `+directive` | ```<!DOCTYPE config system "blah">``` |
+ | `--xml-proc-inst-prefix` | `+p_` | ```<?xml version="1"?>``` |
+
+
+## Encoder / Decoder flag options
+
+In addition to the above flags, there are the following xml encoder/decoder options controlled by flags:
+
+| Flag | Default | Description |
+| -- | -- | -- |
+| `--xml-strict-mode` | false | Strict mode enforces the requirements of the XML specification. When switched off the parser allows input containing common mistakes. See [the Golang xml decoder ](https://pkg.go.dev/encoding/xml#Decoder) for more details.|
+| `--xml-keep-namespace` | true | Keeps the namespace of attributes |
+| `--xml-raw-token` | true | Does not verify that start and end elements match and does not translate name space prefixes to their corresponding URLs. |
+| `--xml-skip-proc-inst` | false | Skips over processing instructions, e.g. `<?xml version="1"?>` |
+| `--xml-skip-directives` | false | Skips over directives, e.g. ```<!DOCTYPE config system "blah">``` |
+
+
+See below for examples
diff --git a/pkg/yqlib/doc/usage/xml.md b/pkg/yqlib/doc/usage/xml.md
index f0d80bf48b..5856f90ab1 100644
--- a/pkg/yqlib/doc/usage/xml.md
+++ b/pkg/yqlib/doc/usage/xml.md
@@ -4,7 +4,32 @@ Encode and decode to and from XML. Whitespace is not conserved for round trips -
Consecutive xml nodes with the same name are assumed to be arrays.
-XML content data and attributes are created as fields. This can be controlled by the `'--xml-attribute-prefix` and `--xml-content-name` flags - see below for examples.
+XML content data, attributes processing instructions and directives are all created as plain fields.
+
+This can be controlled by:
+
+| Flag | Default |Sample XML |
+| -- | -- | -- |
+ | `--xml-attribute-prefix` | `+` (changing to `+@` soon) | Legs in ```<cat legs="4"/>``` |
+ | `--xml-content-name` | `+content` | Meow in ```<cat>Meow <fur>true</true></cat>``` |
+ | `--xml-directive-name` | `+directive` | ```<!DOCTYPE config system "blah">``` |
+ | `--xml-proc-inst-prefix` | `+p_` | ```<?xml version="1"?>``` |
+
+
+## Encoder / Decoder flag options
+
+In addition to the above flags, there are the following xml encoder/decoder options controlled by flags:
+
+| Flag | Default | Description |
+| -- | -- | -- |
+| `--xml-strict-mode` | false | Strict mode enforces the requirements of the XML specification. When switched off the parser allows input containing common mistakes. See [the Golang xml decoder ](https://pkg.go.dev/encoding/xml#Decoder) for more details.|
+| `--xml-keep-namespace` | true | Keeps the namespace of attributes |
+| `--xml-raw-token` | true | Does not verify that start and end elements match and does not translate name space prefixes to their corresponding URLs. |
+| `--xml-skip-proc-inst` | false | Skips over processing instructions, e.g. `<?xml version="1"?>` |
+| `--xml-skip-directives` | false | Skips over directives, e.g. ```<!DOCTYPE config system "blah">``` |
+
+
+See below for examples
{% hint style="warning" %}
Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
@@ -101,7 +126,7 @@ will output
```yaml
+p_xml: version="1.0" encoding="UTF-8"
cat:
- +@legs: "4"
+ +legs: "4"
legs: "7"
```
@@ -122,7 +147,7 @@ will output
+p_xml: version="1.0" encoding="UTF-8"
cat:
+content: meow
- +@legs: "4"
+ +legs: "4"
```
## Parse xml: custom dtd
@@ -320,7 +345,7 @@ Fields with the matching xml-attribute-prefix are assumed to be attributes.
Given a sample.yml file of:
```yaml
cat:
- +@name: tiger
+ +name: tiger
meows: true
```
@@ -341,7 +366,7 @@ Fields with the matching xml-content-name is assumed to be content.
Given a sample.yml file of:
```yaml
cat:
- +@name: tiger
+ +name: tiger
+content: cool
```
diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go
index 0c69a24eca..af78d76e9a 100644
--- a/pkg/yqlib/xml_test.go
+++ b/pkg/yqlib/xml_test.go
@@ -58,7 +58,7 @@ cat:
d:
# in d before
z:
- +@sweet: cool
+ +sweet: cool
# in d after
# in y after
# in_cat_after
@@ -98,11 +98,11 @@ cat:
d:
- # in d before
z:
- +@sweet: cool
+ +sweet: cool
# in d after
- # in d2 before
z:
- +@sweet: cool2
+ +sweet: cool2
# in d2 after
# in y after
# in_cat_after
@@ -161,16 +161,16 @@ const inputXMLWithNamespacedAttr = `
const expectedYAMLWithNamespacedAttr = `+p_xml: version="1.0"
map:
- +@xmlns: some-namespace
- +@xmlns:xsi: some-instance
- +@some-instance:schemaLocation: some-url
+ +xmlns: some-namespace
+ +xmlns:xsi: some-instance
+ +some-instance:schemaLocation: some-url
`
const expectedYAMLWithRawNamespacedAttr = `+p_xml: version="1.0"
map:
- +@xmlns: some-namespace
- +@xmlns:xsi: some-instance
- +@xsi:schemaLocation: some-url
+ +xmlns: some-namespace
+ +xmlns:xsi: some-instance
+ +xsi:schemaLocation: some-url
`
const xmlWithCustomDtd = `
@@ -193,8 +193,9 @@ const expectedDtd = `<?xml version="1.0"?>
</root>
`
-const expectedSkippedDtd = `<root>
- <item>&writer;©right;</item>
+const expectedSkippedDtd = `<?xml version="1.0"?>
+<root>
+ <item>&writer;&copyright;</item>
</root>
`
@@ -227,32 +228,32 @@ var xmlScenarios = []formatScenario{
description: "Parse xml: simple",
subdescription: "Notice how all the values are strings, see the next example on how you can fix that.",
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>\n <says>meow</says>\n <legs>4</legs>\n <cute>true</cute>\n</cat>",
- expected: "cat:\n says: meow\n legs: \"4\"\n cute: \"true\"\n",
+ expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n says: meow\n legs: \"4\"\n cute: \"true\"\n",
},
{
description: "Parse xml: number",
subdescription: "All values are assumed to be strings when parsing XML, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.",
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>\n <says>meow</says>\n <legs>4</legs>\n <cute>true</cute>\n</cat>",
expression: " (.. | select(tag == \"!!str\")) |= from_yaml",
- expected: "cat:\n says: meow\n legs: 4\n cute: true\n",
+ expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n says: meow\n legs: 4\n cute: true\n",
},
{
description: "Parse xml: array",
subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>cat</animal>\n<animal>goat</animal>",
- expected: "animal:\n - cat\n - goat\n",
+ expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\nanimal:\n - cat\n - goat\n",
},
{
description: "Parse xml: attributes",
subdescription: "Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.",
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n <legs>7</legs>\n</cat>",
- expected: "cat:\n +legs: \"4\"\n legs: \"7\"\n",
+ expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +legs: \"4\"\n legs: \"7\"\n",
},
{
description: "Parse xml: attributes with content",
subdescription: "Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own.",
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
- expected: "cat:\n +content: meow\n +legs: \"4\"\n",
+ expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +content: meow\n +legs: \"4\"\n",
},
{
description: "Parse xml: custom dtd",
@@ -362,21 +363,21 @@ var xmlScenarios = []formatScenario{
{
description: "Encode xml: attributes",
subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
- input: "cat:\n +@name: tiger\n meows: true\n",
+ input: "cat:\n +name: tiger\n meows: true\n",
expected: "<cat name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
scenarioType: "encode",
},
{
description: "double prefix",
skipDoc: true,
- input: "cat:\n +@+@name: tiger\n meows: true\n",
+ input: "cat:\n ++@name: tiger\n meows: true\n",
expected: "<cat +@name=\"tiger\">\n <meows>true</meows>\n</cat>\n",
scenarioType: "encode",
},
{
description: "Encode xml: attributes with content",
subdescription: "Fields with the matching xml-content-name is assumed to be content.",
- input: "cat:\n +@name: tiger\n +content: cool\n",
+ input: "cat:\n +name: tiger\n +content: cool\n",
expected: "<cat name=\"tiger\">cool</cat>\n",
scenarioType: "encode",
},
From eb491e05111c6d605fb8416f632b042df8460d7f Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sun, 23 Oct 2022 14:58:04 +1100
Subject: [PATCH 08/10] Removing old warning, added xml update warning
---
README.md | 21 ---------------------
pkg/yqlib/doc/notification-snippet.md | 6 ------
pkg/yqlib/doc/usage/headers/xml.md | 8 ++++++++
3 files changed, 8 insertions(+), 27 deletions(-)
diff --git a/README.md b/README.md
index 1d48743bda..70c0ae9300 100644
--- a/README.md
+++ b/README.md
@@ -7,27 +7,6 @@ a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses
yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below.
-## Notice for v4.x versions prior to 4.18.1
-Since 4.18.1, yq's 'eval/e' command is the _default_ command and no longer needs to be specified.
-
-Older versions will still need to specify 'eval/e'.
-
-Similarly, '-' is no longer required as a filename to read from STDIN (unless reading from one or more files).
-
-TLDR:
-
-Prior to 4.18.1
-```bash
-yq e '.cool' - < file.yaml
-```
-
-4.18+
-```bash
-yq '.cool' < file.yaml
-```
-
-When merging multiple files together, `eval-all/ea` is still required to tell `yq` to run the expression against all the document at once.
-
## Quick Usage Guide
Read a value:
diff --git a/pkg/yqlib/doc/notification-snippet.md b/pkg/yqlib/doc/notification-snippet.md
index 73e76835ff..e69de29bb2 100644
--- a/pkg/yqlib/doc/notification-snippet.md
+++ b/pkg/yqlib/doc/notification-snippet.md
@@ -1,6 +0,0 @@
-
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
diff --git a/pkg/yqlib/doc/usage/headers/xml.md b/pkg/yqlib/doc/usage/headers/xml.md
index ce29869b21..bcd9ba4b3d 100644
--- a/pkg/yqlib/doc/usage/headers/xml.md
+++ b/pkg/yqlib/doc/usage/headers/xml.md
@@ -16,6 +16,14 @@ This can be controlled by:
| `--xml-proc-inst-prefix` | `+p_` | ```<?xml version="1"?>``` |
+{% hint style="warning" %}
+Default Attribute Prefix will be changing in v4.30!
+In order to avoid name conflicts (e.g. having an attribute named "content" will create a field that clashes with the default content name of "+content") the attribute prefix will be changing to "+@".
+
+This will affect users that have not set their own prefix and are not roundtripping XML changes.
+
+{% endhint %}
+
## Encoder / Decoder flag options
In addition to the above flags, there are the following xml encoder/decoder options controlled by flags:
From e4b57694772ba68c293be3e591a135eab0cbf93e Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sun, 23 Oct 2022 14:59:21 +1100
Subject: [PATCH 09/10] Removing old warning, added xml update warning
---
cmd/root.go | 2 +-
pkg/yqlib/doc/operators/add.md | 6 ------
.../doc/operators/alternative-default-value.md | 6 ------
.../doc/operators/anchor-and-alias-operators.md | 6 ------
pkg/yqlib/doc/operators/assign-update.md | 6 ------
pkg/yqlib/doc/operators/boolean-operators.md | 6 ------
pkg/yqlib/doc/operators/collect-into-array.md | 6 ------
pkg/yqlib/doc/operators/column.md | 6 ------
pkg/yqlib/doc/operators/comment-operators.md | 6 ------
pkg/yqlib/doc/operators/compare.md | 6 ------
pkg/yqlib/doc/operators/contains.md | 6 ------
.../doc/operators/create-collect-into-object.md | 6 ------
pkg/yqlib/doc/operators/datetime.md | 6 ------
pkg/yqlib/doc/operators/delete.md | 6 ------
pkg/yqlib/doc/operators/document-index.md | 6 ------
pkg/yqlib/doc/operators/encode-decode.md | 6 ------
pkg/yqlib/doc/operators/entries.md | 6 ------
pkg/yqlib/doc/operators/env-variable-operators.md | 6 ------
pkg/yqlib/doc/operators/equals.md | 6 ------
pkg/yqlib/doc/operators/error.md | 6 ------
pkg/yqlib/doc/operators/eval.md | 6 ------
pkg/yqlib/doc/operators/file-operators.md | 6 ------
pkg/yqlib/doc/operators/flatten.md | 6 ------
pkg/yqlib/doc/operators/group-by.md | 6 ------
pkg/yqlib/doc/operators/has.md | 6 ------
pkg/yqlib/doc/operators/keys.md | 6 ------
pkg/yqlib/doc/operators/length.md | 6 ------
pkg/yqlib/doc/operators/line.md | 6 ------
pkg/yqlib/doc/operators/load.md | 6 ------
pkg/yqlib/doc/operators/map.md | 6 ------
pkg/yqlib/doc/operators/multiply-merge.md | 6 ------
pkg/yqlib/doc/operators/parent.md | 6 ------
pkg/yqlib/doc/operators/path.md | 6 ------
pkg/yqlib/doc/operators/pick.md | 6 ------
pkg/yqlib/doc/operators/pipe.md | 6 ------
pkg/yqlib/doc/operators/recursive-descent-glob.md | 6 ------
pkg/yqlib/doc/operators/reduce.md | 6 ------
pkg/yqlib/doc/operators/reverse.md | 6 ------
pkg/yqlib/doc/operators/select.md | 6 ------
pkg/yqlib/doc/operators/sort-keys.md | 6 ------
pkg/yqlib/doc/operators/sort.md | 6 ------
pkg/yqlib/doc/operators/split-into-documents.md | 6 ------
pkg/yqlib/doc/operators/string-operators.md | 6 ------
pkg/yqlib/doc/operators/style.md | 6 ------
pkg/yqlib/doc/operators/subtract.md | 6 ------
pkg/yqlib/doc/operators/tag.md | 6 ------
pkg/yqlib/doc/operators/traverse-read.md | 6 ------
pkg/yqlib/doc/operators/union.md | 6 ------
pkg/yqlib/doc/operators/unique.md | 6 ------
pkg/yqlib/doc/operators/variable-operators.md | 6 ------
pkg/yqlib/doc/operators/with.md | 6 ------
pkg/yqlib/doc/usage/convert.md | 6 ------
pkg/yqlib/doc/usage/csv-tsv.md | 6 ------
pkg/yqlib/doc/usage/properties.md | 6 ------
pkg/yqlib/doc/usage/xml.md | 14 ++++++++------
55 files changed, 9 insertions(+), 325 deletions(-)
diff --git a/cmd/root.go b/cmd/root.go
index 3a8a762158..df74010450 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -55,7 +55,7 @@ yq -P sample.json
if (inputFormat == "x" || inputFormat == "xml") &&
outputFormat != "x" && outputFormat != "xml" &&
yqlib.XMLPreferences.AttributePrefix == "+" {
- yqlib.GetLogger().Warning("The default xml-attribute-prefix will change in the next release to `+@` to avoid " +
+ yqlib.GetLogger().Warning("The default xml-attribute-prefix will change in the v4.30 to `+@` to avoid " +
"naming conflicts with the default content name, directive name and proc inst prefix. If you need to keep " +
"`+` please set that value explicityly with --xml-attribute-prefix.")
}
diff --git a/pkg/yqlib/doc/operators/add.md b/pkg/yqlib/doc/operators/add.md
index e6c3e099d2..ecadb84f91 100644
--- a/pkg/yqlib/doc/operators/add.md
+++ b/pkg/yqlib/doc/operators/add.md
@@ -9,12 +9,6 @@ Add behaves differently according to the type of the LHS:
Use `+=` as a relative append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Concatenate arrays
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/alternative-default-value.md b/pkg/yqlib/doc/operators/alternative-default-value.md
index 1c70d0cbc8..127465eace 100644
--- a/pkg/yqlib/doc/operators/alternative-default-value.md
+++ b/pkg/yqlib/doc/operators/alternative-default-value.md
@@ -2,12 +2,6 @@
This operator is used to provide alternative (or default) values when a particular expression is either null or false.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## LHS is defined
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md
index 08204a519f..8dbfd996ee 100644
--- a/pkg/yqlib/doc/operators/anchor-and-alias-operators.md
+++ b/pkg/yqlib/doc/operators/anchor-and-alias-operators.md
@@ -5,12 +5,6 @@ Use the `alias` and `anchor` operators to read and write yaml aliases and anchor
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Merge one map
see https://yaml.org/type/merge.html
diff --git a/pkg/yqlib/doc/operators/assign-update.md b/pkg/yqlib/doc/operators/assign-update.md
index af2d80c3c2..5885efc9aa 100644
--- a/pkg/yqlib/doc/operators/assign-update.md
+++ b/pkg/yqlib/doc/operators/assign-update.md
@@ -12,12 +12,6 @@ This will do a similar thing to the plain form, however, the RHS expression is r
### Flags
- `c` clobber custom tags
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Create yaml file
Running
```bash
diff --git a/pkg/yqlib/doc/operators/boolean-operators.md b/pkg/yqlib/doc/operators/boolean-operators.md
index a9f5ddab44..c409445ad0 100644
--- a/pkg/yqlib/doc/operators/boolean-operators.md
+++ b/pkg/yqlib/doc/operators/boolean-operators.md
@@ -16,12 +16,6 @@ These are most commonly used with the `select` operator to filter particular nod
- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## `or` example
Running
```bash
diff --git a/pkg/yqlib/doc/operators/collect-into-array.md b/pkg/yqlib/doc/operators/collect-into-array.md
index 14c2bda06f..73152e515c 100644
--- a/pkg/yqlib/doc/operators/collect-into-array.md
+++ b/pkg/yqlib/doc/operators/collect-into-array.md
@@ -3,12 +3,6 @@
This creates an array using the expression between the square brackets.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Collect empty
Running
```bash
diff --git a/pkg/yqlib/doc/operators/column.md b/pkg/yqlib/doc/operators/column.md
index 3902933a06..d2cef94482 100644
--- a/pkg/yqlib/doc/operators/column.md
+++ b/pkg/yqlib/doc/operators/column.md
@@ -2,12 +2,6 @@
Returns the column of the matching node. Starts from 1, 0 indicates there was no column data.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Returns column of _value_ node
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/comment-operators.md b/pkg/yqlib/doc/operators/comment-operators.md
index 5560b1e9b4..e4dd23ed73 100644
--- a/pkg/yqlib/doc/operators/comment-operators.md
+++ b/pkg/yqlib/doc/operators/comment-operators.md
@@ -10,12 +10,6 @@ This will assign the LHS nodes comments to the expression on the RHS. The RHS is
### relative form: `|=`
Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Set line comment
Set the comment on the key node for more reliability (see below).
diff --git a/pkg/yqlib/doc/operators/compare.md b/pkg/yqlib/doc/operators/compare.md
index e096930fcc..fbd1b1e18f 100644
--- a/pkg/yqlib/doc/operators/compare.md
+++ b/pkg/yqlib/doc/operators/compare.md
@@ -14,12 +14,6 @@ The following types are currently supported:
- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)
- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Compare numbers (>)
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/contains.md b/pkg/yqlib/doc/operators/contains.md
index 4295a0b55b..046b4ca271 100644
--- a/pkg/yqlib/doc/operators/contains.md
+++ b/pkg/yqlib/doc/operators/contains.md
@@ -2,12 +2,6 @@
This returns `true` if the context contains the passed in parameter, and false otherwise.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Array contains array
Array is equal or subset of
diff --git a/pkg/yqlib/doc/operators/create-collect-into-object.md b/pkg/yqlib/doc/operators/create-collect-into-object.md
index 12802ba525..2c132bb83d 100644
--- a/pkg/yqlib/doc/operators/create-collect-into-object.md
+++ b/pkg/yqlib/doc/operators/create-collect-into-object.md
@@ -2,12 +2,6 @@
This is used to construct objects (or maps). This can be used against existing yaml, or to create fresh yaml documents.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Collect empty object
Running
```bash
diff --git a/pkg/yqlib/doc/operators/datetime.md b/pkg/yqlib/doc/operators/datetime.md
index f97a37eab9..845cf1f419 100644
--- a/pkg/yqlib/doc/operators/datetime.md
+++ b/pkg/yqlib/doc/operators/datetime.md
@@ -25,12 +25,6 @@ Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/t
You can durations to time using the `+` operator.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Format: from standard RFC3339 format
Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.
diff --git a/pkg/yqlib/doc/operators/delete.md b/pkg/yqlib/doc/operators/delete.md
index 9eecca59a7..6d0a99a514 100644
--- a/pkg/yqlib/doc/operators/delete.md
+++ b/pkg/yqlib/doc/operators/delete.md
@@ -2,12 +2,6 @@
Deletes matching entries in maps or arrays.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Delete entry in map
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/document-index.md b/pkg/yqlib/doc/operators/document-index.md
index ffb0936dcd..619fd3912c 100644
--- a/pkg/yqlib/doc/operators/document-index.md
+++ b/pkg/yqlib/doc/operators/document-index.md
@@ -2,12 +2,6 @@
Use the `documentIndex` operator (or the `di` shorthand) to select nodes of a particular document.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Retrieve a document index
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/encode-decode.md b/pkg/yqlib/doc/operators/encode-decode.md
index 8a2352d368..71a1863c0d 100644
--- a/pkg/yqlib/doc/operators/encode-decode.md
+++ b/pkg/yqlib/doc/operators/encode-decode.md
@@ -25,12 +25,6 @@ XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify a
Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a string.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Encode value as json string
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/entries.md b/pkg/yqlib/doc/operators/entries.md
index b9d1ae9d0b..a10ad0ef40 100644
--- a/pkg/yqlib/doc/operators/entries.md
+++ b/pkg/yqlib/doc/operators/entries.md
@@ -2,12 +2,6 @@
Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## to_entries Map
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/env-variable-operators.md b/pkg/yqlib/doc/operators/env-variable-operators.md
index a397302320..c616f66c4c 100644
--- a/pkg/yqlib/doc/operators/env-variable-operators.md
+++ b/pkg/yqlib/doc/operators/env-variable-operators.md
@@ -30,12 +30,6 @@ yq '(.. | select(tag == "!!str")) |= envsubst' file.yaml
```
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Read string environment variable
Running
```bash
diff --git a/pkg/yqlib/doc/operators/equals.md b/pkg/yqlib/doc/operators/equals.md
index 4126bf9df3..ca5099b7b9 100644
--- a/pkg/yqlib/doc/operators/equals.md
+++ b/pkg/yqlib/doc/operators/equals.md
@@ -21,12 +21,6 @@ The not equals `!=` operator returns `false` if the LHS is equal to the RHS.
- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Match string
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/error.md b/pkg/yqlib/doc/operators/error.md
index 74a8671e16..0b7a2470fa 100644
--- a/pkg/yqlib/doc/operators/error.md
+++ b/pkg/yqlib/doc/operators/error.md
@@ -2,12 +2,6 @@
Use this operation to short-circuit expressions. Useful for validation.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Validate a particular value
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/eval.md b/pkg/yqlib/doc/operators/eval.md
index eae33bd1fb..1a8a010de9 100644
--- a/pkg/yqlib/doc/operators/eval.md
+++ b/pkg/yqlib/doc/operators/eval.md
@@ -6,12 +6,6 @@ Use `eval` to dynamically process an expression - for instance from an environme
Tip: This can be useful way parameterise complex scripts.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Dynamically evaluate a path
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/file-operators.md b/pkg/yqlib/doc/operators/file-operators.md
index fdfb9e3d8a..b1dacd78dd 100644
--- a/pkg/yqlib/doc/operators/file-operators.md
+++ b/pkg/yqlib/doc/operators/file-operators.md
@@ -10,12 +10,6 @@ Note the use of eval-all to ensure all documents are loaded into memory.
yq eval-all 'select(fi == 0) * select(filename == "file2.yaml")' file1.yaml file2.yaml
```
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Get filename
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/flatten.md b/pkg/yqlib/doc/operators/flatten.md
index f4533646ea..cb3bc26862 100644
--- a/pkg/yqlib/doc/operators/flatten.md
+++ b/pkg/yqlib/doc/operators/flatten.md
@@ -1,12 +1,6 @@
# Flatten
This recursively flattens arrays.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Flatten
Recursively flattens all arrays
diff --git a/pkg/yqlib/doc/operators/group-by.md b/pkg/yqlib/doc/operators/group-by.md
index fc5bb2542c..f5169b72f9 100644
--- a/pkg/yqlib/doc/operators/group-by.md
+++ b/pkg/yqlib/doc/operators/group-by.md
@@ -2,12 +2,6 @@
This is used to group items in an array by an expression.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Group by field
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/has.md b/pkg/yqlib/doc/operators/has.md
index 007ce28924..4a5e294a08 100644
--- a/pkg/yqlib/doc/operators/has.md
+++ b/pkg/yqlib/doc/operators/has.md
@@ -2,12 +2,6 @@
This is operation that returns true if the key exists in a map (or index in an array), false otherwise.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Has map key
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/keys.md b/pkg/yqlib/doc/operators/keys.md
index f4ef293aaa..4dec429bef 100644
--- a/pkg/yqlib/doc/operators/keys.md
+++ b/pkg/yqlib/doc/operators/keys.md
@@ -2,12 +2,6 @@
Use the `keys` operator to return map keys or array indices.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Map keys
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/length.md b/pkg/yqlib/doc/operators/length.md
index 947a1b6e3f..6315e5b2bd 100644
--- a/pkg/yqlib/doc/operators/length.md
+++ b/pkg/yqlib/doc/operators/length.md
@@ -2,12 +2,6 @@
Returns the lengths of the nodes. Length is defined according to the type of the node.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## String length
returns length of string
diff --git a/pkg/yqlib/doc/operators/line.md b/pkg/yqlib/doc/operators/line.md
index dcbf6b6eb0..9fc4cd0c08 100644
--- a/pkg/yqlib/doc/operators/line.md
+++ b/pkg/yqlib/doc/operators/line.md
@@ -2,12 +2,6 @@
Returns the line of the matching node. Starts from 1, 0 indicates there was no line data.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Returns line of _value_ node
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/load.md b/pkg/yqlib/doc/operators/load.md
index 0b2afb730f..79dc014972 100644
--- a/pkg/yqlib/doc/operators/load.md
+++ b/pkg/yqlib/doc/operators/load.md
@@ -45,12 +45,6 @@ this.is = a properties file
bXkgc2VjcmV0IGNoaWxsaSByZWNpcGUgaXMuLi4u
```
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Simple example
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/map.md b/pkg/yqlib/doc/operators/map.md
index 5555b1a865..991167e27f 100644
--- a/pkg/yqlib/doc/operators/map.md
+++ b/pkg/yqlib/doc/operators/map.md
@@ -2,12 +2,6 @@
Maps values of an array. Use `map_values` to map values of an object.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Map array
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/multiply-merge.md b/pkg/yqlib/doc/operators/multiply-merge.md
index c203e3574a..6db1266030 100644
--- a/pkg/yqlib/doc/operators/multiply-merge.md
+++ b/pkg/yqlib/doc/operators/multiply-merge.md
@@ -36,12 +36,6 @@ By default - `yq` merge is naive. It merges maps when they match the key name, a
For more complex array merging (e.g. merging items that match on a certain key) please see the example [here](https://mikefarah.gitbook.io/yq/operators/multiply-merge#merge-arrays-of-objects-together-matching-on-a-key)
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Multiply integers
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/parent.md b/pkg/yqlib/doc/operators/parent.md
index 60cc1529d5..0ef18d55d5 100644
--- a/pkg/yqlib/doc/operators/parent.md
+++ b/pkg/yqlib/doc/operators/parent.md
@@ -2,12 +2,6 @@
Parent simply returns the parent nodes of the matching nodes.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Simple example
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/path.md b/pkg/yqlib/doc/operators/path.md
index 51ac3d4f92..30c910b3e3 100644
--- a/pkg/yqlib/doc/operators/path.md
+++ b/pkg/yqlib/doc/operators/path.md
@@ -7,12 +7,6 @@ You can get the key/index of matching nodes by using the `path` operator to retu
Use `setpath` to set a value to the path array returned by `path`, and similarly `delpaths` for an array of path arrays.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Map path
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/pick.md b/pkg/yqlib/doc/operators/pick.md
index b93bf04f27..45bccc8b8f 100644
--- a/pkg/yqlib/doc/operators/pick.md
+++ b/pkg/yqlib/doc/operators/pick.md
@@ -4,12 +4,6 @@ Filter a map by the specified list of keys. Map is returned with the key in the
Similarly, filter an array by the specified list of indices.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Pick keys from map
Note that the order of the keys matches the pick order and non existent keys are skipped.
diff --git a/pkg/yqlib/doc/operators/pipe.md b/pkg/yqlib/doc/operators/pipe.md
index 21aa380799..0821554ae1 100644
--- a/pkg/yqlib/doc/operators/pipe.md
+++ b/pkg/yqlib/doc/operators/pipe.md
@@ -2,12 +2,6 @@
Pipe the results of an expression into another. Like the bash operator.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Simple Pipe
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/recursive-descent-glob.md b/pkg/yqlib/doc/operators/recursive-descent-glob.md
index 45c6052b73..1bb1656c39 100644
--- a/pkg/yqlib/doc/operators/recursive-descent-glob.md
+++ b/pkg/yqlib/doc/operators/recursive-descent-glob.md
@@ -19,12 +19,6 @@ For instance to set the `style` of all nodes in a yaml doc, including the map ke
```bash
yq '... style= "flow"' file.yaml
```
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Recurse map (values only)
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/reduce.md b/pkg/yqlib/doc/operators/reduce.md
index e06a2a237a..80affbd924 100644
--- a/pkg/yqlib/doc/operators/reduce.md
+++ b/pkg/yqlib/doc/operators/reduce.md
@@ -21,12 +21,6 @@ Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn'
To that end, the reduce operator is called `ireduce` for backwards compatibility if a `jq` like prefix version of `reduce` is ever added.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Sum numbers
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/reverse.md b/pkg/yqlib/doc/operators/reverse.md
index eb9cafc330..77f4037f81 100644
--- a/pkg/yqlib/doc/operators/reverse.md
+++ b/pkg/yqlib/doc/operators/reverse.md
@@ -2,12 +2,6 @@
Reverses the order of the items in an array
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Reverse
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/select.md b/pkg/yqlib/doc/operators/select.md
index 1f1fdb4fd8..67b91f852c 100644
--- a/pkg/yqlib/doc/operators/select.md
+++ b/pkg/yqlib/doc/operators/select.md
@@ -8,12 +8,6 @@ Select is used to filter arrays and maps by a boolean expression.
- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Select elements from array using wildcard prefix
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/sort-keys.md b/pkg/yqlib/doc/operators/sort-keys.md
index b5101dc117..1be4c99894 100644
--- a/pkg/yqlib/doc/operators/sort-keys.md
+++ b/pkg/yqlib/doc/operators/sort-keys.md
@@ -12,12 +12,6 @@ diff file1.yml file2.yml
Note that `yq` does not yet consider anchors when sorting by keys - this may result in invalid yaml documents if your are using merge anchors.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Sort keys of map
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/sort.md b/pkg/yqlib/doc/operators/sort.md
index b7b810d399..54111e8d01 100644
--- a/pkg/yqlib/doc/operators/sort.md
+++ b/pkg/yqlib/doc/operators/sort.md
@@ -7,12 +7,6 @@ To sort by descending order, pipe the results through the `reverse` operator aft
Note that at this stage, `yq` only sorts scalar fields.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Sort by string field
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/split-into-documents.md b/pkg/yqlib/doc/operators/split-into-documents.md
index a4f22f3df5..e072328f3d 100644
--- a/pkg/yqlib/doc/operators/split-into-documents.md
+++ b/pkg/yqlib/doc/operators/split-into-documents.md
@@ -2,12 +2,6 @@
This operator splits all matches into separate documents
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Split empty
Running
```bash
diff --git a/pkg/yqlib/doc/operators/string-operators.md b/pkg/yqlib/doc/operators/string-operators.md
index 3882a67a90..585404cf27 100644
--- a/pkg/yqlib/doc/operators/string-operators.md
+++ b/pkg/yqlib/doc/operators/string-operators.md
@@ -56,12 +56,6 @@ IFS= read -rd '' output < <(cat my_file)
output=$output ./yq '.data.values = strenv(output)' first.yml
```
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## To up (upper) case
Works with unicode characters
diff --git a/pkg/yqlib/doc/operators/style.md b/pkg/yqlib/doc/operators/style.md
index e4f4a924b3..b21a5b266f 100644
--- a/pkg/yqlib/doc/operators/style.md
+++ b/pkg/yqlib/doc/operators/style.md
@@ -2,12 +2,6 @@
The style operator can be used to get or set the style of nodes (e.g. string style, yaml style)
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Update and set style of a particular node (simple)
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/subtract.md b/pkg/yqlib/doc/operators/subtract.md
index a8f8d33848..b850cc9761 100644
--- a/pkg/yqlib/doc/operators/subtract.md
+++ b/pkg/yqlib/doc/operators/subtract.md
@@ -2,12 +2,6 @@
You can use subtract to subtract numbers, as well as removing elements from an array.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Array subtraction
Running
```bash
diff --git a/pkg/yqlib/doc/operators/tag.md b/pkg/yqlib/doc/operators/tag.md
index 33e11f65a5..43db143924 100644
--- a/pkg/yqlib/doc/operators/tag.md
+++ b/pkg/yqlib/doc/operators/tag.md
@@ -2,12 +2,6 @@
The tag operator can be used to get or set the tag of nodes (e.g. `!!str`, `!!int`, `!!bool`).
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Get tag
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/traverse-read.md b/pkg/yqlib/doc/operators/traverse-read.md
index f8c07b8d02..3a4452e3b1 100644
--- a/pkg/yqlib/doc/operators/traverse-read.md
+++ b/pkg/yqlib/doc/operators/traverse-read.md
@@ -2,12 +2,6 @@
This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structures.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Simple map navigation
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/union.md b/pkg/yqlib/doc/operators/union.md
index 6a477698c5..abf0befeb8 100644
--- a/pkg/yqlib/doc/operators/union.md
+++ b/pkg/yqlib/doc/operators/union.md
@@ -2,12 +2,6 @@
This operator is used to combine different results together.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Combine scalars
Running
```bash
diff --git a/pkg/yqlib/doc/operators/unique.md b/pkg/yqlib/doc/operators/unique.md
index c7bd23ba8d..a1bc443c61 100644
--- a/pkg/yqlib/doc/operators/unique.md
+++ b/pkg/yqlib/doc/operators/unique.md
@@ -3,12 +3,6 @@
This is used to filter out duplicated items in an array. Note that the original order of the array is maintained.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Unique array of scalars (string/numbers)
Note that unique maintains the original order of the array.
diff --git a/pkg/yqlib/doc/operators/variable-operators.md b/pkg/yqlib/doc/operators/variable-operators.md
index e7efcd0582..5c9a293d74 100644
--- a/pkg/yqlib/doc/operators/variable-operators.md
+++ b/pkg/yqlib/doc/operators/variable-operators.md
@@ -4,12 +4,6 @@ Like the `jq` equivalents, variables are sometimes required for the more complex
Note that there is also an additional `ref` operator that holds a reference (instead of a copy) of the path, allowing you to make multiple changes to the same path.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Single value variable
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/operators/with.md b/pkg/yqlib/doc/operators/with.md
index 9804cfe980..6b1a913fc2 100644
--- a/pkg/yqlib/doc/operators/with.md
+++ b/pkg/yqlib/doc/operators/with.md
@@ -2,12 +2,6 @@
Use the `with` operator to conveniently make multiple updates to a deeply nested path, or to update array elements relatively to each other. The first argument expression sets the root context, and the second expression runs against that root context.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Update and style
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/usage/convert.md b/pkg/yqlib/doc/usage/convert.md
index f915945747..1f149c11ba 100644
--- a/pkg/yqlib/doc/usage/convert.md
+++ b/pkg/yqlib/doc/usage/convert.md
@@ -5,12 +5,6 @@ Encode and decode to and from JSON. Supports multiple JSON documents in a single
Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Parse json: simple
JSON is a subset of yaml, so all you need to do is prettify the output
diff --git a/pkg/yqlib/doc/usage/csv-tsv.md b/pkg/yqlib/doc/usage/csv-tsv.md
index 03749ab636..c58e42cde2 100644
--- a/pkg/yqlib/doc/usage/csv-tsv.md
+++ b/pkg/yqlib/doc/usage/csv-tsv.md
@@ -29,12 +29,6 @@ Fifi,cat
```
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Encode CSV simple
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/doc/usage/properties.md b/pkg/yqlib/doc/usage/properties.md
index f80bba35d2..e7952fcfdb 100644
--- a/pkg/yqlib/doc/usage/properties.md
+++ b/pkg/yqlib/doc/usage/properties.md
@@ -4,12 +4,6 @@ Encode/Decode/Roundtrip to/from a property file. Line comments on value nodes wi
By default, empty maps and arrays are not encoded - see below for an example on how to encode a value for these.
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Encode properties
Note that empty arrays and maps are not encoded by default.
diff --git a/pkg/yqlib/doc/usage/xml.md b/pkg/yqlib/doc/usage/xml.md
index 5856f90ab1..714194c387 100644
--- a/pkg/yqlib/doc/usage/xml.md
+++ b/pkg/yqlib/doc/usage/xml.md
@@ -16,6 +16,14 @@ This can be controlled by:
| `--xml-proc-inst-prefix` | `+p_` | ```<?xml version="1"?>``` |
+{% hint style="warning" %}
+Default Attribute Prefix will be changing in v4.30!
+In order to avoid name conflicts (e.g. having an attribute named "content" will create a field that clashes with the default content name of "+content") the attribute prefix will be changing to "+@".
+
+This will affect users that have not set their own prefix and are not roundtripping XML changes.
+
+{% endhint %}
+
## Encoder / Decoder flag options
In addition to the above flags, there are the following xml encoder/decoder options controlled by flags:
@@ -31,12 +39,6 @@ In addition to the above flags, there are the following xml encoder/decoder opti
See below for examples
-{% hint style="warning" %}
-Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
-
-`yq e <exp> <file>`
-{% endhint %}
-
## Parse xml: simple
Notice how all the values are strings, see the next example on how you can fix that.
From 86ef4e4d909ab9230b54dd3d0d10a6151b2ae0fc Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Mon, 24 Oct 2022 09:54:36 +1100
Subject: [PATCH 10/10] Updated XML acceptance tests with proc inst and
directives
---
acceptance_tests/inputs-format.sh | 25 +++++++++++++++++++++++--
1 file changed, 23 insertions(+), 2 deletions(-)
diff --git a/acceptance_tests/inputs-format.sh b/acceptance_tests/inputs-format.sh
index 2173678e2a..8979d04d97 100755
--- a/acceptance_tests/inputs-format.sh
+++ b/acceptance_tests/inputs-format.sh
@@ -127,6 +127,7 @@ testInputXmlNamespaces() {
EOL
read -r -d '' expected << EOM
++p_xml: version="1.0"
map:
+xmlns: some-namespace
+xmlns:xsi: some-instance
@@ -140,6 +141,26 @@ EOM
assertEquals "$expected" "$X"
}
+testInputXmlRoundtrip() {
+ cat >test.yml <<EOL
+<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
+EOL
+
+ read -r -d '' expected << EOM
+<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
+EOM
+
+ X=$(./yq -p=xml -o=xml test.yml)
+ assertEquals "$expected" "$X"
+
+ X=$(./yq ea -p=xml -o=xml test.yml)
+ assertEquals "$expected" "$X"
+}
+
testInputXmlStrict() {
cat >test.yml <<EOL
@@ -153,11 +174,11 @@ testInputXmlStrict() {
</root>
EOL
- X=$(./yq -p=xml --xml-strict-mode test.yml 2>&1)
+ X=$(./yq -p=xml --xml-strict-mode test.yml -o=xml 2>&1)
assertEquals 1 $?
assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"
- X=$(./yq ea -p=xml --xml-strict-mode test.yml 2>&1)
+ X=$(./yq ea -p=xml --xml-strict-mode test.yml -o=xml 2>&1)
assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"
}