diff --git a/cmd/root.go b/cmd/root.go index 96f9815d8f..45cb9de6e3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -78,6 +78,11 @@ yq -P sample.json rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. )") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. )") + rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocPrefix, "lua-prefix", yqlib.ConfiguredLuaPreferences.DocPrefix, "prefix") + rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocSuffix, "lua-suffix", yqlib.ConfiguredLuaPreferences.DocSuffix, "suffix") + rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})") + rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables") + rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.") rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)") diff --git a/cmd/utils.go b/cmd/utils.go index 7d34c723ae..a50f7048bd 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -197,6 +197,8 @@ func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) { return yqlib.NewTomlEncoder(), nil case yqlib.ShellVariablesOutputFormat: return yqlib.NewShellVariablesEncoder(), nil + case yqlib.LuaOutputFormat: + return yqlib.NewLuaEncoder(yqlib.ConfiguredLuaPreferences), nil } return nil, fmt.Errorf("invalid encoder: %v", format) } diff --git a/pkg/yqlib/doc/usage/lua.md b/pkg/yqlib/doc/usage/lua.md new file mode 100644 index 0000000000..321417077c --- /dev/null +++ b/pkg/yqlib/doc/usage/lua.md @@ -0,0 +1,144 @@ + +## Basic example +Given a sample.yml file of: +```yaml +--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth +``` +then +```bash +yq -o=lua '.' sample.yml +``` +will output +```lua +return { + ["country"] = "Australia"; -- this place + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +``` + +## Unquoted keys +Uses the `--lua-unquoted` option to produce a nicer-looking output. + +Given a sample.yml file of: +```yaml +--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth +``` +then +```bash +yq -o=lua '.' sample.yml +``` +will output +```lua +return { + country = "Australia"; -- this place + cities = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +``` + +## Globals +Uses the `--lua-globals` option to export the values into the global scope. + +Given a sample.yml file of: +```yaml +--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth +``` +then +```bash +yq -o=lua '.' sample.yml +``` +will output +```lua +country = "Australia"; -- this place +cities = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", +}; +``` + +## Elaborate example +Given a sample.yml file of: +```yaml +--- +hello: world +tables: + like: this + keys: values + ? look: non-string keys + : True +numbers: + - decimal: 12345 + - hex: 0x7fabc123 + - octal: 0o30 + - float: 123.45 + - infinity: .inf + - not: .nan + +``` +then +```bash +yq -o=lua '.' sample.yml +``` +will output +```lua +return { + ["hello"] = "world"; + ["tables"] = { + ["like"] = "this"; + ["keys"] = "values"; + [{ + ["look"] = "non-string keys"; + }] = true; + }; + ["numbers"] = { + { + ["decimal"] = 12345; + }, + { + ["hex"] = 0x7fabc123; + }, + { + ["octal"] = 24; + }, + { + ["float"] = 123.45; + }, + { + ["infinity"] = (1/0); + }, + { + ["not"] = (0/0); + }, + }; +}; +``` + diff --git a/pkg/yqlib/encoder_lua.go b/pkg/yqlib/encoder_lua.go new file mode 100644 index 0000000000..898b3855ef --- /dev/null +++ b/pkg/yqlib/encoder_lua.go @@ -0,0 +1,328 @@ +package yqlib + +import ( + "fmt" + "io" + "strings" + + yaml "gopkg.in/yaml.v3" +) + +type luaEncoder struct { + docPrefix string + docSuffix string + indent int + indentStr string + unquoted bool + globals bool + escape *strings.Replacer +} + +func (le *luaEncoder) CanHandleAliases() bool { + return false +} + +func NewLuaEncoder(prefs LuaPreferences) Encoder { + escape := strings.NewReplacer( + "\000", "\\000", + "\001", "\\001", + "\002", "\\002", + "\003", "\\003", + "\004", "\\004", + "\005", "\\005", + "\006", "\\006", + "\007", "\\a", + "\010", "\\b", + "\011", "\\t", + "\012", "\\n", + "\013", "\\v", + "\014", "\\f", + "\015", "\\r", + "\016", "\\014", + "\017", "\\015", + "\020", "\\016", + "\021", "\\017", + "\022", "\\018", + "\023", "\\019", + "\024", "\\020", + "\025", "\\021", + "\026", "\\022", + "\027", "\\023", + "\030", "\\024", + "\031", "\\025", + "\032", "\\026", + "\033", "\\027", + "\034", "\\028", + "\035", "\\029", + "\036", "\\030", + "\037", "\\031", + "\"", "\\\"", + "'", "\\'", + "\\", "\\\\", + "\177", "\\127", + ) + unescape := strings.NewReplacer( + "\\'", "'", + "\\\"", "\"", + "\\n", "\n", + "\\r", "\r", + "\\t", "\t", + "\\\\", "\\", + ) + return &luaEncoder{unescape.Replace(prefs.DocPrefix), unescape.Replace(prefs.DocSuffix), 0, "\t", prefs.UnquotedKeys, prefs.Globals, escape} +} + +func (le *luaEncoder) PrintDocumentSeparator(writer io.Writer) error { + return nil +} + +func (le *luaEncoder) PrintLeadingContent(writer io.Writer, content string) error { + return nil +} + +func (le *luaEncoder) encodeString(writer io.Writer, node *yaml.Node) error { + quote := "\"" + switch node.Style { + case yaml.LiteralStyle, yaml.FoldedStyle, yaml.FlowStyle: + for i := 0; i < 10; i++ { + if !strings.Contains(node.Value, "]"+strings.Repeat("=", i)+"]") { + err := writeString(writer, "["+strings.Repeat("=", i)+"[\n") + if err != nil { + return err + } + err = writeString(writer, node.Value) + if err != nil { + return err + } + return writeString(writer, "]"+strings.Repeat("=", i)+"]") + } + } + case yaml.SingleQuotedStyle: + quote = "'" + + // falltrough to regular ol' string + } + return writeString(writer, quote+le.escape.Replace(node.Value)+quote) +} + +func (le *luaEncoder) writeIndent(writer io.Writer) error { + if le.indentStr == "" { + return nil + } + err := writeString(writer, "\n") + if err != nil { + return err + } + return writeString(writer, strings.Repeat(le.indentStr, le.indent)) +} + +func (le *luaEncoder) encodeArray(writer io.Writer, node *yaml.Node) error { + err := writeString(writer, "{") + if err != nil { + return err + } + le.indent++ + for _, child := range node.Content { + err = le.writeIndent(writer) + if err != nil { + return err + } + err := le.Encode(writer, child) + if err != nil { + return err + } + err = writeString(writer, ",") + if err != nil { + return err + } + if child.LineComment != "" { + sansPrefix, _ := strings.CutPrefix(child.LineComment, "#") + err = writeString(writer, " --"+sansPrefix) + if err != nil { + return err + } + } + } + le.indent-- + if len(node.Content) != 0 { + err = le.writeIndent(writer) + if err != nil { + return err + } + } + return writeString(writer, "}") +} + +func needsQuoting(s string) bool { + // known keywords as of Lua 5.4 + switch s { + case "do", "and", "else", "break", + "if", "end", "goto", "false", + "in", "for", "then", "local", + "or", "nil", "true", "until", + "elseif", "function", "not", + "repeat", "return", "while": + return true + } + // [%a_][%w_]* + for i, c := range s { + if i == 0 { + if !((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_') { + return true + } + } else { + if !((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_') { + return true + } + } + } + return false +} + +func (le *luaEncoder) encodeMap(writer io.Writer, node *yaml.Node, global bool) error { + if !global { + err := writeString(writer, "{") + if err != nil { + return err + } + le.indent++ + } + for i, child := range node.Content { + if (i % 2) == 1 { + // value + err := le.Encode(writer, child) + if err != nil { + return err + } + err = writeString(writer, ";") + if err != nil { + return err + } + } else { + // key + if !global || i > 0 { + err := le.writeIndent(writer) + if err != nil { + return err + } + } + if (le.unquoted || global) && child.Tag == "!!str" && !needsQuoting(child.Value) { + err := writeString(writer, child.Value+" = ") + if err != nil { + return err + } + } else { + if global { + // This only works in Lua 5.2+ + err := writeString(writer, "_ENV") + if err != nil { + return err + } + } + err := writeString(writer, "[") + if err != nil { + return err + } + err = le.encodeAny(writer, child) + if err != nil { + return err + } + err = writeString(writer, "] = ") + if err != nil { + return err + } + } + } + if child.LineComment != "" { + sansPrefix, _ := strings.CutPrefix(child.LineComment, "#") + err := writeString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix) + if err != nil { + return err + } + if (i % 2) == 0 { + // newline and indent after comments on keys + err = le.writeIndent(writer) + if err != nil { + return err + } + } + } + } + if global { + return writeString(writer, "\n") + } + le.indent-- + if len(node.Content) != 0 { + err := le.writeIndent(writer) + if err != nil { + return err + } + } + return writeString(writer, "}") +} + +func (le *luaEncoder) encodeAny(writer io.Writer, node *yaml.Node) error { + switch node.Kind { + case yaml.SequenceNode: + return le.encodeArray(writer, node) + case yaml.MappingNode: + return le.encodeMap(writer, node, false) + case yaml.ScalarNode: + switch node.Tag { + case "!!str": + return le.encodeString(writer, node) + case "!!null": + // TODO reject invalid use as a table key + return writeString(writer, "nil") + case "!!bool": + // Yaml 1.2 has case variation e.g. True, FALSE etc but Lua only has + // lower case + return writeString(writer, strings.ToLower(node.Value)) + case "!!int": + if strings.HasPrefix(node.Value, "0o") { + var octalValue int + err := node.Decode(&octalValue) + if err != nil { + return err + } + return writeString(writer, fmt.Sprintf("%d", octalValue)) + } + return writeString(writer, strings.ToLower(node.Value)) + case "!!float": + switch strings.ToLower(node.Value) { + case ".inf": + return writeString(writer, "(1/0)") + case "-.inf": + return writeString(writer, "(-1/0)") + case ".nan": + return writeString(writer, "(0/0)") + default: + return writeString(writer, node.Value) + } + default: + return fmt.Errorf("Lua encoder NYI -- %s", node.ShortTag()) + } + case yaml.DocumentNode: + if le.globals { + if node.Content[0].Kind != yaml.MappingNode { + return fmt.Errorf("--lua-global requires a top level MappingNode") + } + return le.encodeMap(writer, node.Content[0], true) + } + err := writeString(writer, le.docPrefix) + if err != nil { + return err + } + err = le.encodeAny(writer, node.Content[0]) + if err != nil { + return err + } + return writeString(writer, le.docSuffix) + default: + return fmt.Errorf("Lua encoder NYI -- %s", node.ShortTag()) + } +} + +func (le *luaEncoder) Encode(writer io.Writer, node *yaml.Node) error { + return le.encodeAny(writer, node) +} diff --git a/pkg/yqlib/lua.go b/pkg/yqlib/lua.go new file mode 100644 index 0000000000..ba63e63492 --- /dev/null +++ b/pkg/yqlib/lua.go @@ -0,0 +1,19 @@ +package yqlib + +type LuaPreferences struct { + DocPrefix string + DocSuffix string + UnquotedKeys bool + Globals bool +} + +func NewDefaultLuaPreferences() LuaPreferences { + return LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: false, + Globals: false, + } +} + +var ConfiguredLuaPreferences = NewDefaultLuaPreferences() diff --git a/pkg/yqlib/lua_test.go b/pkg/yqlib/lua_test.go new file mode 100644 index 0000000000..3f8133d659 --- /dev/null +++ b/pkg/yqlib/lua_test.go @@ -0,0 +1,257 @@ +package yqlib + +import ( + "bufio" + "fmt" + "testing" + + "github.com/mikefarah/yq/v4/test" +) + +var luaScenarios = []formatScenario{ + { + description: "Basic example", + scenarioType: "encode", + input: `--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth`, + expected: `return { + ["country"] = "Australia"; -- this place + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +`, + }, + { + description: "Unquoted keys", + subdescription: "Uses the `--lua-unquoted` option to produce a nicer-looking output.", + scenarioType: "unquoted-encode", + input: `--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth`, + expected: `return { + country = "Australia"; -- this place + cities = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +`, + }, + { + description: "Globals", + subdescription: "Uses the `--lua-globals` option to export the values into the global scope.", + scenarioType: "globals-encode", + input: `--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth`, + expected: `country = "Australia"; -- this place +cities = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", +}; +`, + }, + { + description: "Elaborate example", + input: `--- +hello: world +tables: + like: this + keys: values + ? look: non-string keys + : True +numbers: + - decimal: 12345 + - hex: 0x7fabc123 + - octal: 0o30 + - float: 123.45 + - infinity: .inf + - not: .nan +`, + expected: `return { + ["hello"] = "world"; + ["tables"] = { + ["like"] = "this"; + ["keys"] = "values"; + [{ + ["look"] = "non-string keys"; + }] = true; + }; + ["numbers"] = { + { + ["decimal"] = 12345; + }, + { + ["hex"] = 0x7fabc123; + }, + { + ["octal"] = 24; + }, + { + ["float"] = 123.45; + }, + { + ["infinity"] = (1/0); + }, + { + ["not"] = (0/0); + }, + }; +}; +`, + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Sequence", + input: "- a\n- b\n- c\n", + expected: "return {\n\t\"a\",\n\t\"b\",\n\t\"c\",\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Mapping", + input: "a: b\nc:\n d: e\nf: 0\n", + expected: "return {\n\t[\"a\"] = \"b\";\n\t[\"c\"] = {\n\t\t[\"d\"] = \"e\";\n\t};\n\t[\"f\"] = 0;\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Scalar str", + input: "str: |\n foo\n bar\nanother: 'single'\nand: \"double\"", + expected: "return {\n\t[\"str\"] = [[\nfoo\nbar\n]];\n\t[\"another\"] = 'single';\n\t[\"and\"] = \"double\";\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Scalar null", + input: "x: null\n", + expected: "return {\n\t[\"x\"] = nil;\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Scalar int", + input: "- 1\n- 2\n- 0x10\n- 0o30\n- -999\n", + expected: "return {\n\t1,\n\t2,\n\t0x10,\n\t24,\n\t-999,\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Scalar float", + input: "- 1.0\n- 3.14\n- 1e100\n- .Inf\n- .NAN\n", + expected: "return {\n\t1.0,\n\t3.14,\n\t1e100,\n\t(1/0),\n\t(0/0),\n};\n", + scenarioType: "encode", + }, +} + +func testLuaScenario(t *testing.T, s formatScenario) { + switch s.scenarioType { + case "encode": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description) + case "unquoted-encode": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: true, + Globals: false, + })), s.description) + case "globals-encode": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: false, + Globals: true, + })), s.description) + default: + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) + } +} + +func documentLuaScenario(t *testing.T, w *bufio.Writer, i interface{}) { + s := i.(formatScenario) + + if s.skipDoc { + return + } + switch s.scenarioType { + case "encode", "unquoted-encode", "globals-encode": + documentLuaEncodeScenario(w, s) + default: + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) + } +} + +func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) { + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + + prefs := ConfiguredLuaPreferences + switch s.scenarioType { + case "unquoted-encode": + prefs = LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: true, + Globals: false, + } + case "globals-encode": + prefs = LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: false, + Globals: true, + } + } + writeOrPanic(w, "Given a sample.yml file of:\n") + writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) + + writeOrPanic(w, "then\n") + switch s.scenarioType { + case "unquoted-encode": + writeOrPanic(w, "```bash\nyq -o=lua --lua-unquoted '.' sample.yml\n```\n") + case "globals-encode": + writeOrPanic(w, "```bash\nyq -o=lua --lua-globals '.' sample.yml\n```\n") + default: + writeOrPanic(w, "```bash\nyq -o=lua '.' sample.yml\n```\n") + } + writeOrPanic(w, "will output\n") + + writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(prefs)))) +} + +func TestLuaScenarios(t *testing.T) { + for _, tt := range luaScenarios { + testLuaScenario(t, tt) + } + genericScenarios := make([]interface{}, len(luaScenarios)) + for i, s := range luaScenarios { + genericScenarios[i] = s + } + documentScenarios(t, "usage", "lua", genericScenarios, documentLuaScenario) +} diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index 9cd65753f9..c430fcfedf 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -33,6 +33,7 @@ const ( ShOutputFormat TomlOutputFormat ShellVariablesOutputFormat + LuaOutputFormat ) func OutputFormatFromString(format string) (PrinterOutputFormat, error) { @@ -53,8 +54,10 @@ func OutputFormatFromString(format string) (PrinterOutputFormat, error) { return TomlOutputFormat, nil case "shell", "s", "sh": return ShellVariablesOutputFormat, nil + case "lua", "l": + return LuaOutputFormat, nil default: - return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml|shell]", format) + return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml|shell|lua]", format) } }