From 30017d116d83b329e8019a571022f2d3a5f0ca3f Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Tue, 17 Aug 2021 11:20:43 +0200 Subject: [PATCH 1/9] Rename fq gojq fork to github.com/wader/gojq --- README.md | 2 +- _tools/gen_builtin.go | 2 +- _tools/print_builtin.go | 2 +- cli/cli.go | 2 +- cli/inputs.go | 2 +- cmd/gojq/main.go | 2 +- compare_test.go | 2 +- compiler_test.go | 2 +- encoder_test.go | 2 +- go.dev.mod | 2 +- go.mod | 4 +++- option_environ_loader_test.go | 2 +- option_function_test.go | 2 +- option_input_iter_test.go | 2 +- option_iter_function_test.go | 2 +- option_module_loader_test.go | 2 +- option_test.go | 2 +- option_variables_test.go | 2 +- query_test.go | 2 +- 19 files changed, 21 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index ca13b2f6..18215b81 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ import ( "fmt" "log" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) func main() { diff --git a/_tools/gen_builtin.go b/_tools/gen_builtin.go index 3b0d3300..eaeedaa8 100644 --- a/_tools/gen_builtin.go +++ b/_tools/gen_builtin.go @@ -11,7 +11,7 @@ import ( "strings" "github.com/itchyny/astgen-go" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) const fileFormat = `// Code generated by _tools/gen_builtin.go; DO NOT EDIT. diff --git a/_tools/print_builtin.go b/_tools/print_builtin.go index 8169a309..e73d4772 100644 --- a/_tools/print_builtin.go +++ b/_tools/print_builtin.go @@ -7,7 +7,7 @@ import ( "sort" "strings" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) func main() { diff --git a/cli/cli.go b/cli/cli.go index 28fdfed2..a1da10ad 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -12,7 +12,7 @@ import ( "github.com/mattn/go-isatty" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) const name = "gojq" diff --git a/cli/inputs.go b/cli/inputs.go index 1f91c480..3c5b6ea0 100644 --- a/cli/inputs.go +++ b/cli/inputs.go @@ -10,7 +10,7 @@ import ( "gopkg.in/yaml.v3" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) type inputReader struct { diff --git a/cmd/gojq/main.go b/cmd/gojq/main.go index 0fded1d1..0df030a9 100644 --- a/cmd/gojq/main.go +++ b/cmd/gojq/main.go @@ -4,7 +4,7 @@ package main import ( "os" - "github.com/itchyny/gojq/cli" + "github.com/wader/gojq/cli" ) func main() { diff --git a/compare_test.go b/compare_test.go index f05e9904..3bdce2e3 100644 --- a/compare_test.go +++ b/compare_test.go @@ -6,7 +6,7 @@ import ( "math/big" "testing" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) func TestCompare(t *testing.T) { diff --git a/compiler_test.go b/compiler_test.go index d12240aa..e1860d07 100644 --- a/compiler_test.go +++ b/compiler_test.go @@ -11,7 +11,7 @@ import ( "time" "unsafe" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) func ExampleCompile() { diff --git a/encoder_test.go b/encoder_test.go index 1d1247eb..22c450af 100644 --- a/encoder_test.go +++ b/encoder_test.go @@ -6,7 +6,7 @@ import ( "math/big" "testing" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) func TestMarshal(t *testing.T) { diff --git a/go.dev.mod b/go.dev.mod index 9a0579ca..cb2410e8 100644 --- a/go.dev.mod +++ b/go.dev.mod @@ -1,4 +1,4 @@ -module github.com/itchyny/gojq +module github.com/wader/gojq go 1.18 diff --git a/go.mod b/go.mod index 88e73667..4ce518d1 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,6 @@ -module github.com/itchyny/gojq +// fork of ithub.com/itchyny/gojq +// renames module to make it easier to depend on without using replace +module github.com/wader/gojq go 1.18 diff --git a/option_environ_loader_test.go b/option_environ_loader_test.go index 4447d4cd..cb12af30 100644 --- a/option_environ_loader_test.go +++ b/option_environ_loader_test.go @@ -4,7 +4,7 @@ import ( "fmt" "log" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) func ExampleWithEnvironLoader() { diff --git a/option_function_test.go b/option_function_test.go index fe13ff9a..174d3024 100644 --- a/option_function_test.go +++ b/option_function_test.go @@ -7,7 +7,7 @@ import ( "math/big" "strconv" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) func toFloat(x any) (float64, bool) { diff --git a/option_input_iter_test.go b/option_input_iter_test.go index c0510d0a..9b30a7ea 100644 --- a/option_input_iter_test.go +++ b/option_input_iter_test.go @@ -4,7 +4,7 @@ import ( "fmt" "log" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) func ExampleWithInputIter() { diff --git a/option_iter_function_test.go b/option_iter_function_test.go index a7ffc771..f8b49496 100644 --- a/option_iter_function_test.go +++ b/option_iter_function_test.go @@ -4,7 +4,7 @@ import ( "fmt" "log" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) // Implementation of range/2 using WithIterFunction option. diff --git a/option_module_loader_test.go b/option_module_loader_test.go index 7b41d51a..623fa688 100644 --- a/option_module_loader_test.go +++ b/option_module_loader_test.go @@ -4,7 +4,7 @@ import ( "fmt" "log" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) type moduleLoader struct{} diff --git a/option_test.go b/option_test.go index a1bc581a..2a3a3d8d 100644 --- a/option_test.go +++ b/option_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) func TestWithModuleLoaderError(t *testing.T) { diff --git a/option_variables_test.go b/option_variables_test.go index 0d4accc4..7fdae84e 100644 --- a/option_variables_test.go +++ b/option_variables_test.go @@ -4,7 +4,7 @@ import ( "fmt" "log" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) func ExampleWithVariables() { diff --git a/query_test.go b/query_test.go index 87c4b527..e2065d21 100644 --- a/query_test.go +++ b/query_test.go @@ -15,7 +15,7 @@ import ( "testing" "time" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) func ExampleQuery_Run() { From e65a7df38f70338b9b0fe30eff35636226be1b12 Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Mon, 11 Jan 2021 21:23:00 +0100 Subject: [PATCH 2/9] Add binary, octal and hex litrals --- lexer.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++ normalize.go | 2 +- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/lexer.go b/lexer.go index 82bb2b6b..57a462e0 100644 --- a/lexer.go +++ b/lexer.go @@ -75,6 +75,21 @@ func (l *lexer) Lex(lval *yySymType) (tokenType int) { } return tokIdent case isNumber(ch): + if ch == '0' { + base := l.peek() + switch base { + case 'b', 'o', 'x': + i := l.offset - 1 + l.offset++ + for isInteger(base, l.peek()) { + l.offset++ + } + l.token = string(l.source[i:l.offset]) + lval.token = l.token + return tokNumber + } + } + i := l.offset - 1 j := l.scanNumber(numberStateLead) if j < 0 { @@ -376,6 +391,21 @@ func (l *lexer) validNumber() bool { ch = l.peek() state = numberStateFloat } + + switch ch { + case '0': + l.offset++ + base := l.peek() + switch base { + case 'b', 'o', 'x': + l.offset++ + for isInteger(base, l.peek()) { + l.offset++ + } + return l.offset == len(l.source) + } + } + return isNumber(ch) && l.scanNumber(state) == len(l.source) } @@ -559,10 +589,34 @@ func isHex(ch byte) bool { isNumber(ch) } +func isBinary(ch byte) bool { + return '0' == ch || ch == '1' +} + +func isOctal(ch byte) bool { + return '0' <= ch && ch <= '7' +} + func isNumber(ch byte) bool { return '0' <= ch && ch <= '9' } +func isInteger(base byte, ch byte) bool { + if ch == '_' { + return true + } + switch base { + case 'b': + return isBinary(ch) + case 'o': + return isOctal(ch) + case 'x': + return isHex(ch) + default: + panic("unreachable") + } +} + func isNewLine(ch byte) bool { switch ch { case '\n', '\r': diff --git a/normalize.go b/normalize.go index 2bfcd215..f9a6cde1 100644 --- a/normalize.go +++ b/normalize.go @@ -16,7 +16,7 @@ func normalizeNumber(v json.Number) any { return f } } - if bi, ok := new(big.Int).SetString(v.String(), 10); ok { + if bi, ok := new(big.Int).SetString(v.String(), 0); ok { return bi } if strings.HasPrefix(v.String(), "-") { From 8c8958e37699ccd911acc246272ffdd32cce71fc Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Mon, 1 Feb 2021 19:38:24 +0100 Subject: [PATCH 3/9] Add scope function to get all functions and varialbes in scope --- compiler.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ func.go | 11 +++++++++++ 2 files changed, 55 insertions(+) diff --git a/compiler.go b/compiler.go index de5f9a10..54947c12 100644 --- a/compiler.go +++ b/compiler.go @@ -973,6 +973,13 @@ func (c *compiler) compileFunc(e *Func) error { true, -1, ) + case "scope": + return c.compileCallInternal( + [3]any{c.funcScope, 0, e.Name}, + e.Args, + true, + -1, + ) case "input": if c.inputIter == nil { return &inputNotAllowedError{} @@ -1161,6 +1168,43 @@ func (c *compiler) funcBuiltins(any, []any) any { return ys } +func (c *compiler) funcScope(any, []any) any { + var xs []any + for _, fds := range builtinFuncDefs { + for _, fd := range fds { + xs = append(xs, fmt.Sprintf("%s/%d", fd.Name, len(fd.Args))) + } + } + for name, f := range internalFuncs { + for _, a := range f.arities() { + xs = append(xs, fmt.Sprintf("%s/%d", name, a)) + } + } + for name, f := range c.customFuncs { + for _, a := range f.arities() { + xs = append(xs, fmt.Sprintf("%s/%d", name, a)) + } + } + if c.environLoader != nil { + xs = append(xs, "$ENV") + } + for i := len(c.scopes) - 1; i >= 0; i-- { + s := c.scopes[i] + for j := len(s.variables) - 1; j >= 0; j-- { + v := s.variables[j] + xs = append(xs, v.name) + } + for j := len(s.funcs) - 1; j >= 0; j-- { + f := s.funcs[j] + xs = append(xs, fmt.Sprintf("%s/%d", f.name, f.argcnt)) + } + } + sort.Slice(xs, func(i, j int) bool { + return xs[i].(string) < xs[j].(string) + }) + return xs +} + func (c *compiler) funcInput(any, []any) any { v, ok := c.inputIter.Next() if !ok { diff --git a/func.go b/func.go index 98b6a5cd..bbc9cc51 100644 --- a/func.go +++ b/func.go @@ -40,6 +40,16 @@ func (fn function) accept(cnt int) bool { return fn.argcount&(1< 0; i, cnt = i+1, cnt>>1 { + if cnt&1 > 0 { + as = append(as, i) + } + } + return as +} + var internalFuncs map[string]function func init() { @@ -48,6 +58,7 @@ func init() { "path": argFunc1(nil), "env": argFunc0(nil), "builtins": argFunc0(nil), + "scope": argFunc0(nil), "input": argFunc0(nil), "modulemeta": argFunc0(nil), "length": argFunc0(funcLength), From d650fafbfb82d8068be313369d6e6138c4ba1ad4 Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Sat, 13 Aug 2022 21:06:16 +0200 Subject: [PATCH 4/9] Add expermental `raw string` support --- lexer.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lexer.go b/lexer.go index 57a462e0..e00f69b2 100644 --- a/lexer.go +++ b/lexer.go @@ -247,6 +247,10 @@ func (l *lexer) Lex(lval *yySymType) (tokenType int) { tok, str := l.scanString(l.offset - 1) lval.token = str return tok + case '`': + tok, str := l.scanRawString(l.offset - 1) + lval.token = str + return tok default: if ch >= utf8.RuneSelf { r, size := utf8.DecodeRuneInString(l.source[l.offset-1:]) @@ -508,6 +512,21 @@ func (l *lexer) scanString(start int) (int, string) { return tokUnterminatedString, "" } +func (l *lexer) scanRawString(start int) (int, string) { + for i := l.offset; i < len(l.source); i++ { + ch := l.source[i] + switch ch { + case '`': + l.offset = i + 1 + l.token = l.source[start:l.offset] + return tokString, l.token[1 : len(l.token)-1] + } + } + l.offset = len(l.source) + l.token = "" + return tokUnterminatedString, "" +} + func quoteAndEscape(src string, quote bool, controls int) []byte { size := len(src) + controls*5 if quote { From 9de05c361213baa03a772f29afb1de49fd3c20e9 Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Sat, 6 Feb 2021 16:26:09 +0100 Subject: [PATCH 5/9] WIP: add scopedump function --- compiler.go | 14 ++++++++++++++ func.go | 1 + 2 files changed, 15 insertions(+) diff --git a/compiler.go b/compiler.go index 54947c12..336de472 100644 --- a/compiler.go +++ b/compiler.go @@ -997,6 +997,20 @@ func (c *compiler) compileFunc(e *Func) error { true, -1, ) + case "scopedump": + c.append(&code{op: oppop}) + n := 0 + for i := len(c.scopes) - 1; i >= 0; i-- { + s := c.scopes[i] + for j := len(s.variables) - 1; j >= 0; j-- { + v := s.variables[j] + c.append(&code{op: oppush, v: v.name}) + c.append(&code{op: opload, v: v.index}) + n++ + } + } + c.append(&code{op: opobject, v: n}) + return nil default: return c.compileCall(e.Name, e.Args) } diff --git a/func.go b/func.go index bbc9cc51..067bbfcd 100644 --- a/func.go +++ b/func.go @@ -59,6 +59,7 @@ func init() { "env": argFunc0(nil), "builtins": argFunc0(nil), "scope": argFunc0(nil), + "scopedump": argFunc0(nil), "input": argFunc0(nil), "modulemeta": argFunc0(nil), "length": argFunc0(funcLength), From 0f36fc121f35cc7e58a197b2062d31626a52c135 Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Fri, 20 Aug 2021 17:10:02 +0200 Subject: [PATCH 6/9] Query marshal/unmarshal support --- operator.go | 80 +++++++++++++++++++++- query.go | 188 +++++++++++++++++++++++++-------------------------- term_type.go | 74 ++++++++++++++++++++ 3 files changed, 247 insertions(+), 95 deletions(-) diff --git a/operator.go b/operator.go index 73a548e0..5e0d064b 100644 --- a/operator.go +++ b/operator.go @@ -1,6 +1,8 @@ package gojq import ( + "encoding/json" + "fmt" "math" "math/big" "strings" @@ -37,6 +39,62 @@ const ( OpUpdateAlt ) +// String implements [fmt.Stringer]. +func OperatorFromString(s string) Operator { + switch s { + case "|": + return OpPipe + case ",": + return OpComma + case "+": + return OpAdd + case "-": + return OpSub + case "*": + return OpMul + case "/": + return OpDiv + case "%": + return OpMod + case "==": + return OpEq + case "!=": + return OpNe + case ">": + return OpGt + case "<": + return OpLt + case ">=": + return OpGe + case "<=": + return OpLe + case "and": + return OpAnd + case "or": + return OpOr + case "//": + return OpAlt + case "=": + return OpAssign + case "|=": + return OpModify + case "+=": + return OpUpdateAdd + case "-=": + return OpUpdateSub + case "*=": + return OpUpdateMul + case "/=": + return OpUpdateDiv + case "%=": + return OpUpdateMod + case "//=": + return OpUpdateAlt + default: + return 0 + } +} + // String implements [fmt.Stringer]. func (op Operator) String() string { switch op { @@ -89,7 +147,7 @@ func (op Operator) String() string { case OpUpdateAlt: return "//=" default: - panic(op) + return "" } } @@ -207,6 +265,26 @@ func (op Operator) getFunc() string { } } +func (op Operator) MarshalJSON() ([]byte, error) { + if op == 0 { + return json.Marshal(nil) + } + return json.Marshal(op.String()) +} + +func (op *Operator) UnmarshalJSON(text []byte) error { + var s string + err := json.Unmarshal(text, &s) + if err != nil { + return err + } + *op = OperatorFromString(s) + if *op == 0 { + return fmt.Errorf("unknown operator %v", s) + } + return nil +} + func binopTypeSwitch( l, r any, callbackInts func(_, _ int) any, diff --git a/query.go b/query.go index 5f20b4ff..6a7f2b0a 100644 --- a/query.go +++ b/query.go @@ -7,14 +7,14 @@ import ( // Query represents the abstract syntax tree of a jq query. type Query struct { - Meta *ConstObject - Imports []*Import - FuncDefs []*FuncDef - Term *Term - Left *Query - Op Operator - Right *Query - Func string + Meta *ConstObject `json:"meta,omitempty"` + Imports []*Import `json:"imports,omitempty"` + FuncDefs []*FuncDef `json:"func_defs,omitempty"` + Term *Term `json:"term,omitempty"` + Left *Query `json:"left,omitempty"` + Op Operator `json:"op,omitempty"` + Right *Query `json:"right,omitempty"` + Func string `json:"func,omitempty"` } // Run the query. @@ -108,10 +108,10 @@ func (e *Query) toIndices(xs []any) []any { // Import ... type Import struct { - ImportPath string - ImportAlias string - IncludePath string - Meta *ConstObject + ImportPath string `json:"import_path,omitempty"` + ImportAlias string `json:"import_alias,omitempty"` + IncludePath string `json:"include_path,omitempty"` + Meta *ConstObject `json:"meta,omitempty"` } func (e *Import) String() string { @@ -139,9 +139,9 @@ func (e *Import) writeTo(s *strings.Builder) { // FuncDef ... type FuncDef struct { - Name string - Args []string - Body *Query + Name string `json:"name,omitempty"` + Args []string `json:"args,omitempty"` + Body *Query `json:"body,omitempty"` } func (e *FuncDef) String() string { @@ -175,23 +175,23 @@ func (e *FuncDef) Minify() { // Term ... type Term struct { - Type TermType - Index *Index - Func *Func - Object *Object - Array *Array - Number string - Unary *Unary - Format string - Str *String - If *If - Try *Try - Reduce *Reduce - Foreach *Foreach - Label *Label - Break string - Query *Query - SuffixList []*Suffix + Type TermType `json:"type,omitempty"` + Index *Index `json:"index,omitempty"` + Func *Func `json:"func,omitempty"` + Object *Object `json:"object,omitempty"` + Array *Array `json:"array,omitempty"` + Number string `json:"number,omitempty"` + Unary *Unary `json:"unary,omitempty"` + Format string `json:"format,omitempty"` + Str *String `json:"str,omitempty"` + If *If `json:"if,omitempty"` + Try *Try `json:"try,omitempty"` + Reduce *Reduce `json:"reduce,omitempty"` + Foreach *Foreach `json:"foreach,omitempty"` + Label *Label `json:"label,omitempty"` + Break string `json:"break,omitempty"` + Query *Query `json:"query,omitempty"` + SuffixList []*Suffix `json:"suffix_list,omitempty"` } func (e *Term) String() string { @@ -360,8 +360,8 @@ func (e *Term) toNumber() any { // Unary ... type Unary struct { - Op Operator - Term *Term + Op Operator `json:"op,omitempty"` + Term *Term `json:"term,omitempty"` } func (e *Unary) String() string { @@ -389,9 +389,9 @@ func (e *Unary) toNumber() any { // Pattern ... type Pattern struct { - Name string - Array []*Pattern - Object []*PatternObject + Name string `json:"name,omitempty"` + Array []*Pattern `json:"array,omitempty"` + Object []*PatternObject `json:"object,omitempty"` } func (e *Pattern) String() string { @@ -426,10 +426,10 @@ func (e *Pattern) writeTo(s *strings.Builder) { // PatternObject ... type PatternObject struct { - Key string - KeyString *String - KeyQuery *Query - Val *Pattern + Key string `json:"key,omitempty"` + KeyString *String `json:"key_string,omitempty"` + KeyQuery *Query `json:"key_query,omitempty"` + Val *Pattern `json:"val,omitempty"` } func (e *PatternObject) String() string { @@ -456,11 +456,11 @@ func (e *PatternObject) writeTo(s *strings.Builder) { // Index ... type Index struct { - Name string - Str *String - Start *Query - End *Query - IsSlice bool + Name string `json:"name,omitempty"` + Str *String `json:"str,omitempty"` + Start *Query `json:"start,omitempty"` + End *Query `json:"end,omitempty"` + IsSlice bool `json:"is_slice,omitempty"` } func (e *Index) String() string { @@ -550,8 +550,8 @@ func (e *Index) toIndices(xs []any) []any { // Func ... type Func struct { - Name string - Args []*Query + Name string `json:"name,omitempty"` + Args []*Query `json:"args,omitempty"` } func (e *Func) String() string { @@ -589,8 +589,8 @@ func (e *Func) toFunc() string { // String ... type String struct { - Str string - Queries []*Query + Str string `json:"str,omitempty"` + Queries []*Query `json:"queries,omitempty"` } func (e *String) String() string { @@ -625,7 +625,7 @@ func (e *String) minify() { // Object ... type Object struct { - KeyVals []*ObjectKeyVal + KeyVals []*ObjectKeyVal `json:"key_vals,omitempty"` } func (e *Object) String() string { @@ -657,10 +657,10 @@ func (e *Object) minify() { // ObjectKeyVal ... type ObjectKeyVal struct { - Key string - KeyString *String - KeyQuery *Query - Val *ObjectVal + Key string `json:"key,omitempty"` + KeyString *String `json:"key_string,omitempty"` + KeyQuery *Query `json:"key_query,omitempty"` + Val *ObjectVal `json:"val,omitempty"` } func (e *ObjectKeyVal) String() string { @@ -698,7 +698,7 @@ func (e *ObjectKeyVal) minify() { // ObjectVal ... type ObjectVal struct { - Queries []*Query + Queries []*Query `json:"queries,omitempty"` } func (e *ObjectVal) String() string { @@ -724,7 +724,7 @@ func (e *ObjectVal) minify() { // Array ... type Array struct { - Query *Query + Query *Query `json:"query,omitempty"` } func (e *Array) String() string { @@ -749,10 +749,10 @@ func (e *Array) minify() { // Suffix ... type Suffix struct { - Index *Index - Iter bool - Optional bool - Bind *Bind + Index *Index `json:"index,omitempty"` + Iter bool `json:"iter,omitempty"` + Optional bool `json:"optional,omitempty"` + Bind *Bind `json:"bind,omitempty"` } func (e *Suffix) String() string { @@ -804,8 +804,8 @@ func (e *Suffix) toIndices(xs []any) []any { // Bind ... type Bind struct { - Patterns []*Pattern - Body *Query + Patterns []*Pattern `json:"patterns,omitempty"` + Body *Query `json:"body,omitempty"` } func (e *Bind) String() string { @@ -836,10 +836,10 @@ func (e *Bind) minify() { // If ... type If struct { - Cond *Query - Then *Query - Elif []*IfElif - Else *Query + Cond *Query `json:"cond,omitempty"` + Then *Query `json:"then,omitempty"` + Elif []*IfElif `json:"elif,omitempty"` + Else *Query `json:"else,omitempty"` } func (e *If) String() string { @@ -877,8 +877,8 @@ func (e *If) minify() { // IfElif ... type IfElif struct { - Cond *Query - Then *Query + Cond *Query `json:"cond,omitempty"` + Then *Query `json:"then,omitempty"` } func (e *IfElif) String() string { @@ -901,8 +901,8 @@ func (e *IfElif) minify() { // Try ... type Try struct { - Body *Query - Catch *Query + Body *Query `json:"body,omitempty"` + Catch *Query `json:"catch,omitempty"` } func (e *Try) String() string { @@ -929,10 +929,10 @@ func (e *Try) minify() { // Reduce ... type Reduce struct { - Term *Term - Pattern *Pattern - Start *Query - Update *Query + Term *Term `json:"term,omitempty"` + Pattern *Pattern `json:"pattern,omitempty"` + Start *Query `json:"start,omitempty"` + Update *Query `json:"update,omitempty"` } func (e *Reduce) String() string { @@ -961,11 +961,11 @@ func (e *Reduce) minify() { // Foreach ... type Foreach struct { - Term *Term - Pattern *Pattern - Start *Query - Update *Query - Extract *Query + Term *Term `json:"term,omitempty"` + Pattern *Pattern `json:"pattern,omitempty"` + Start *Query `json:"start,omitempty"` + Update *Query `json:"update,omitempty"` + Extract *Query `json:"extract,omitempty"` } func (e *Foreach) String() string { @@ -1001,8 +1001,8 @@ func (e *Foreach) minify() { // Label ... type Label struct { - Ident string - Body *Query + Ident string `json:"ident,omitempty"` + Body *Query `json:"body,omitempty"` } func (e *Label) String() string { @@ -1024,13 +1024,13 @@ func (e *Label) minify() { // ConstTerm ... type ConstTerm struct { - Object *ConstObject - Array *ConstArray - Number string - Str string - Null bool - True bool - False bool + Object *ConstObject `json:"object,omitempty"` + Array *ConstArray `json:"array,omitempty"` + Number string `json:"number,omitempty"` + Str string `json:"str,omitempty"` + Null bool `json:"null,omitempty"` + True bool `json:"true,omitempty"` + False bool `json:"false,omitempty"` } func (e *ConstTerm) String() string { @@ -1077,7 +1077,7 @@ func (e *ConstTerm) toValue() any { // ConstObject ... type ConstObject struct { - KeyVals []*ConstObjectKeyVal + KeyVals []*ConstObjectKeyVal `json:"keyvals,omitempty"` } func (e *ConstObject) String() string { @@ -1119,9 +1119,9 @@ func (e *ConstObject) ToValue() map[string]any { // ConstObjectKeyVal ... type ConstObjectKeyVal struct { - Key string - KeyString string - Val *ConstTerm + Key string `json:"key,omitempty"` + KeyString string `json:"key_string,omitempty"` + Val *ConstTerm `json:"val,omitempty"` } func (e *ConstObjectKeyVal) String() string { @@ -1142,7 +1142,7 @@ func (e *ConstObjectKeyVal) writeTo(s *strings.Builder) { // ConstArray ... type ConstArray struct { - Elems []*ConstTerm + Elems []*ConstTerm `json:"elems,omitempty"` } func (e *ConstArray) String() string { diff --git a/term_type.go b/term_type.go index 941e7ba9..463f6c88 100644 --- a/term_type.go +++ b/term_type.go @@ -1,5 +1,10 @@ package gojq +import ( + "encoding/json" + "fmt" +) + // TermType represents the type of [Term]. type TermType int @@ -27,6 +32,54 @@ const ( TermTypeQuery ) +// GoString implements [fmt.GoStringer]. +func TermTypeFromString(s string) TermType { + switch s { + case "TermTypeIdentity": + return TermTypeIdentity + case "TermTypeRecurse": + return TermTypeRecurse + case "TermTypeNull": + return TermTypeNull + case "TermTypeTrue": + return TermTypeTrue + case "TermTypeFalse": + return TermTypeFalse + case "TermTypeIndex": + return TermTypeIndex + case "TermTypeFunc": + return TermTypeFunc + case "TermTypeObject": + return TermTypeObject + case "TermTypeArray": + return TermTypeArray + case "TermTypeNumber": + return TermTypeNumber + case "TermTypeUnary": + return TermTypeUnary + case "TermTypeFormat": + return TermTypeFormat + case "TermTypeString": + return TermTypeString + case "TermTypeIf": + return TermTypeIf + case "TermTypeTry": + return TermTypeTry + case "TermTypeReduce": + return TermTypeReduce + case "TermTypeForeach": + return TermTypeForeach + case "TermTypeLabel": + return TermTypeLabel + case "TermTypeBreak": + return TermTypeBreak + case "TermTypeQuery": + return TermTypeQuery + default: + return 0 + } +} + // GoString implements [fmt.GoStringer]. func (termType TermType) GoString() (str string) { defer func() { str = "gojq." + str }() @@ -75,3 +128,24 @@ func (termType TermType) GoString() (str string) { panic(termType) } } + +func (termType TermType) MarshalJSON() ([]byte, error) { + if termType == 0 { + return json.Marshal(nil) + } + // TODO: gojq. skips prefix + return json.Marshal(termType.GoString()[5:]) +} + +func (termType *TermType) UnmarshalJSON(text []byte) error { + var s string + err := json.Unmarshal(text, &s) + if err != nil { + return err + } + *termType = TermTypeFromString(s) + if *termType == 0 { + return fmt.Errorf("unknown term %v", s) + } + return nil +} From 3243a9ee4f829fcf16afd235a9bd92c65afe41cf Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Sat, 9 Apr 2022 17:36:14 +0200 Subject: [PATCH 7/9] Export BinopTypeSwitch helper for external binops --- operator.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/operator.go b/operator.go index 5e0d064b..111bc0c3 100644 --- a/operator.go +++ b/operator.go @@ -285,6 +285,29 @@ func (op *Operator) UnmarshalJSON(text []byte) error { return nil } +// BinopTypeSwitch helper for external binops +// re-exported instead of renamed to make it easier to follow upstream +func BinopTypeSwitch( + l, r any, + callbackInts func(_, _ int) any, + callbackFloats func(_, _ float64) any, + callbackBigInts func(_, _ *big.Int) any, + callbackStrings func(_, _ string) any, + callbackArrays func(_, _ []any) any, + callbackMaps func(_, _ map[string]any) any, + fallback func(_, _ any) any) any { + return binopTypeSwitch( + l, r, + callbackInts, + callbackFloats, + callbackBigInts, + callbackStrings, + callbackArrays, + callbackMaps, + fallback, + ) +} + func binopTypeSwitch( l, r any, callbackInts func(_, _ int) any, From b7e613069119f987cdeaad3b65c6ef77abad84a2 Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Sat, 5 Dec 2020 20:19:51 +0100 Subject: [PATCH 8/9] Add JQValue interface to support custom values --- cli/encoder.go | 6 + debug.go | 2 + encoder.go | 2 + execute.go | 39 ++++++- func.go | 304 +++++++++++++++++++++++++++++++++++++++--------- gojq.go | 64 ++++++++++ normalize.go | 12 ++ operator.go | 23 +++- preview_test.go | 2 +- type.go | 7 +- type_test.go | 2 +- 11 files changed, 400 insertions(+), 63 deletions(-) diff --git a/cli/encoder.go b/cli/encoder.go index 8841baac..b54ff50a 100644 --- a/cli/encoder.go +++ b/cli/encoder.go @@ -9,6 +9,8 @@ import ( "sort" "strconv" "unicode/utf8" + + "github.com/wader/gojq" ) type encoder struct { @@ -66,6 +68,10 @@ func (e *encoder) encode(v any) error { if err := e.encodeObject(v); err != nil { return err } + case gojq.JQValue: + if err := e.encode(v.JQValueToGoJQ()); err != nil { + return err + } default: panic(fmt.Sprintf("invalid type: %[1]T (%[1]v)", v)) } diff --git a/debug.go b/debug.go index ad3d7216..10d022c8 100644 --- a/debug.go +++ b/debug.go @@ -198,6 +198,8 @@ func debugValue(v any) string { return fmt.Sprintf("gojq.Iter(%#v)", v) case []pathValue: return fmt.Sprintf("[]gojq.pathValue(%v)", v) + case JQValue: + return fmt.Sprintf("gojq.JQValue(%s)", Preview(v)) case [2]int: return fmt.Sprintf("[%d,%d]", v[0], v[1]) case [3]int: diff --git a/encoder.go b/encoder.go index 3233e8a9..e3c4f0f2 100644 --- a/encoder.go +++ b/encoder.go @@ -68,6 +68,8 @@ func (e *encoder) encode(v any) { e.encodeArray(v) case map[string]any: e.encodeObject(v) + case JQValue: + e.encode(v.JQValueToGoJQ()) default: panic(fmt.Sprintf("invalid type: %[1]T (%[1]v)", v)) } diff --git a/execute.go b/execute.go index dcf9d984..137c768b 100644 --- a/execute.go +++ b/execute.go @@ -61,6 +61,9 @@ loop: m := make(map[string]any, n) for i := 0; i < n; i++ { v, k := env.pop(), env.pop() + if jv, ok := k.(JQValue); ok { + k = jv.JQValueToString() + } s, ok := k.(string) if !ok { err = &objectKeyNotStringError{k} @@ -147,7 +150,9 @@ loop: pc = code.v.(int) goto loop case opjumpifnot: - if v := env.pop(); v == nil || v == false { + v := env.pop() + b, bOk := toBoolean(v) + if isNull(v) || (bOk && !b) { pc = code.v.(int) goto loop } @@ -157,6 +162,9 @@ loop: } p, v := code.v, env.pop() if code.op == opindexarray && v != nil { + if jqv, ok := v.(JQValue); ok { + v = jqv.JQValueToGoJQ() + } if _, ok := v.([]any); !ok { err = &expectedArrayError{v} break loop @@ -320,6 +328,32 @@ loop: continue } break loop + case JQValue: + if !env.paths.empty() && env.expdepth == 0 && !env.pathIntact(v) { + err = &invalidPathIterError{v} + break loop + } + xsv := v.JQValueEach() + if e, ok := xsv.(error); ok { + err = e + break loop + } + switch xsv := xsv.(type) { + case []PathValue: + // convert from external PathValue to internal pathValue to make it easier to follow upstream + xs = make([]pathValue, len(xsv)) + if len(xsv) == 0 { + break loop + } + for i, pv := range xsv { + xs[i] = pathValue{path: pv.Path, value: pv.Value} + } + case nil: + break loop + default: + err = &iteratorError{xsv} + break loop + } default: err = &iteratorError{v} env.push(emptyIter{}) @@ -431,6 +465,9 @@ func (env *env) pathIntact(v any) bool { if w, ok := w.(float64); ok { return v == w || math.IsNaN(v) && math.IsNaN(w) } + case JQValue: + // TODO: JQValue: should understand this better + return true } return v == w } diff --git a/func.go b/func.go index 067bbfcd..3635ff53 100644 --- a/func.go +++ b/func.go @@ -308,13 +308,15 @@ func funcLength(v any) any { return len(v) case map[string]any: return len(v) + case JQValue: + return v.JQValueLength() default: return &func0TypeError{"length", v} } } func funcUtf8ByteLength(v any) any { - s, ok := v.(string) + s, ok := toString(v) if !ok { return &func0TypeError{"utf8bytelength", v} } @@ -335,6 +337,8 @@ func funcKeys(v any) any { w[i] = k } return w + case JQValue: + return v.JQValueKeys() default: return &func0TypeError{"keys", v} } @@ -361,6 +365,8 @@ func values(v any) ([]any, bool) { vs[i] = v[k] } return vs, true + case JQValue: + return values(v.JQValueToGoJQ()) default: return nil, false } @@ -379,6 +385,8 @@ func funcHas(v, x any) any { } case nil: return false + case JQValue: + return v.JQValueHas(x) } return &func1TypeError{"has", v, x} } @@ -397,12 +405,48 @@ func funcToEntries(v any) any { w[i] = map[string]any{"key": k, "value": v[k]} } return w + case JQValue: + // to_entries/0 used to be implemented in jq but was made internal for + // performance. To preserve the JQValue keys order we have to implement + // it ourself, otherwise keys will be sorted. + if v.JQValueType() == JQTypeObject { + lv := v.JQValueLength() + if err, ok := lv.(error); ok { + return err + } + l, ok := toInt(lv) + if !ok { + return fmt.Errorf("invalid int length: %v", lv) + } + ev := v.JQValueEach() + e, ok := ev.([]PathValue) + if !ok { + return &func0TypeError{"to_entries", v} + } + + w := make([]any, l) + for i, pv := range e { + k, ok := pv.Path.(string) + if !ok { + return &func0TypeError{"to_entries", v} + } + + w[i] = map[string]any{"key": k, "value": pv.Value} + } + + return w + } + return funcToEntries(v.JQValueToGoJQ()) default: return &func0TypeError{"to_entries", v} } } func funcFromEntries(v any) any { + if jqv, ok := v.(JQValue); ok { + v = jqv.JQValueToGoJQ() + } + vs, ok := v.([]any) if !ok { return &func0TypeError{"from_entries", v} @@ -418,6 +462,13 @@ func funcFromEntries(v any) any { ) for _, k := range [4]string{"key", "Key", "name", "Name"} { if k := v[k]; k != nil && k != false { + if jqvk, ok := k.(JQValue); ok { + k = jqvk.JQValueToGoJQ() + if k == false { + continue + } + } + if key, ok = k.(string); !ok { return &func0WrapError{"from_entries", vs, &objectKeyNotStringError{k}} } @@ -442,6 +493,7 @@ func funcFromEntries(v any) any { func funcAdd(v any) any { vs, ok := values(v) + if !ok { return &func0TypeError{"add", v} } @@ -511,6 +563,8 @@ func funcToNumber(v any) any { return &func0WrapError{"tonumber", v, errors.New("invalid number")} } return toNumber(v) + case JQValue: + return v.JQValueToNumber() default: return &func0TypeError{"tonumber", v} } @@ -521,7 +575,7 @@ func toNumber(v string) any { } func funcToString(v any) any { - if s, ok := v.(string); ok { + if s, ok := toString(v); ok { return s } return funcToJSON(v) @@ -532,10 +586,11 @@ func funcType(v any) any { } func funcReverse(v any) any { - vs, ok := v.([]any) + vs, ok := toArray(v) if !ok { return &func0TypeError{"reverse", v} } + ws := make([]any, len(vs)) for i, v := range vs { ws[len(ws)-i-1] = v @@ -638,21 +693,23 @@ func indexFunc(v, x any, f func(_, _ []any) any) any { return f(v, []any{x}) } case string: - if x, ok := x.(string); ok { + if x, ok := toString(x); ok { return f(explode(v), explode(x)) } return &expectedStringError{x} + case JQValue: + return indexFunc(v.JQValueToGoJQ(), x, f) default: return &expectedArrayError{v} } } func funcStartsWith(v, x any) any { - s, ok := v.(string) + s, ok := toString(v) if !ok { return &func1TypeError{"startswith", v, x} } - t, ok := x.(string) + t, ok := toString(x) if !ok { return &func1TypeError{"startswith", v, x} } @@ -660,11 +717,11 @@ func funcStartsWith(v, x any) any { } func funcEndsWith(v, x any) any { - s, ok := v.(string) + s, ok := toString(v) if !ok { return &func1TypeError{"endswith", v, x} } - t, ok := x.(string) + t, ok := toString(x) if !ok { return &func1TypeError{"endswith", v, x} } @@ -672,11 +729,11 @@ func funcEndsWith(v, x any) any { } func funcLtrimstr(v, x any) any { - s, ok := v.(string) + s, ok := toString(v) if !ok { return v } - t, ok := x.(string) + t, ok := toString(x) if !ok { return v } @@ -684,11 +741,11 @@ func funcLtrimstr(v, x any) any { } func funcRtrimstr(v, x any) any { - s, ok := v.(string) + s, ok := toString(v) if !ok { return v } - t, ok := x.(string) + t, ok := toString(x) if !ok { return v } @@ -696,11 +753,11 @@ func funcRtrimstr(v, x any) any { } func funcExplode(v any) any { - s, ok := v.(string) + x, ok := toString(v) if !ok { return &func0TypeError{"explode", v} } - return explode(s) + return explode(x) } func explode(s string) []any { @@ -714,7 +771,7 @@ func explode(s string) []any { } func funcImplode(v any) any { - vs, ok := v.([]any) + vs, ok := toArray(v) if !ok { return &func0TypeError{"implode", v} } @@ -731,11 +788,11 @@ func funcImplode(v any) any { } func funcSplit(v any, args []any) any { - s, ok := v.(string) + s, ok := toString(v) if !ok { return &func0TypeError{"split", v} } - x, ok := args[0].(string) + x, ok := toString(args[0]) if !ok { return &func0TypeError{"split", x} } @@ -745,7 +802,7 @@ func funcSplit(v any, args []any) any { } else { var flags string if args[1] != nil { - v, ok := args[1].(string) + v, ok := toString(args[1]) if !ok { return &func0TypeError{"split", args[1]} } @@ -795,7 +852,7 @@ func funcToJSON(v any) any { } func funcFromJSON(v any) any { - s, ok := v.(string) + s, ok := toString(v) if !ok { return &func0TypeError{"fromjson", v} } @@ -812,14 +869,14 @@ func funcFromJSON(v any) any { } func funcFormat(v, x any) any { - s, ok := x.(string) + sx, ok := toString(x) if !ok { return &func0TypeError{"format", x} } - format := "@" + s - f := formatToFunc(format) + fmt := "@" + sx + f := formatToFunc(fmt) if f == nil { - return &formatNotFoundError{format} + return &formatNotFoundError{fmt} } return internalFuncs[f.Name].callback(v, nil) } @@ -892,7 +949,7 @@ var shEscaper = strings.NewReplacer( ) func funcToSh(v any) any { - if _, ok := v.([]any); !ok { + if _, ok := toArray(v); !ok { v = []any{v} } return formatJoin("sh", v, " ", func(s string) string { @@ -901,12 +958,16 @@ func funcToSh(v any) any { } func formatJoin(typ string, v any, sep string, escape func(string) string) any { - vs, ok := v.([]any) + vs, ok := toArray(v) if !ok { return &func0TypeError{"@" + typ, v} } ss := make([]string, len(vs)) for i, v := range vs { + if jqv, ok := v.(JQValue); ok { + v = jqv.JQValueToGoJQ() + } + switch v := v.(type) { case []any, map[string]any: return &formatRowError{typ, v} @@ -947,6 +1008,7 @@ func funcToBase64d(v any) any { } func funcIndex2(_, v, x any) any { + switch x := x.(type) { case string: switch v := v.(type) { @@ -954,6 +1016,8 @@ func funcIndex2(_, v, x any) any { return nil case map[string]any: return v[x] + case JQValue: + return v.JQValueKey(x) default: return &expectedObjectError{v} } @@ -966,6 +1030,22 @@ func funcIndex2(_, v, x any) any { return index(v, i) case string: return indexString(v, i) + case JQValue: + lv := v.JQValueSliceLen() + l, ok := lv.(int) + if !ok { + return lv + } + i := clampIndex(i, -1, l) + + // TODO: JQValue -2 outside < 0, -1 outside > len + // TODO: redo this, nice to know actual index? + if i < 0 { + i = -2 + } else if i >= l { + i = -1 + } + return v.JQValueIndex(i) default: return &expectedArrayError{v} } @@ -991,6 +1071,8 @@ func funcIndex2(_, v, x any) any { return &expectedStartEndError{x} } return funcSlice(nil, v, end, start) + case JQValue: + return funcIndex2(nil, v, x.JQValueToGoJQ()) default: switch v.(type) { case []any: @@ -1032,6 +1114,8 @@ func funcSlice(_, v, e, s any) (r any) { return slice(v, e, s) case string: return sliceString(v, e, s) + case JQValue: + return sliceJQValue(v, e, s) default: return &expectedArrayError{v} } @@ -1100,6 +1184,33 @@ func sliceString(v string, e, s any) any { return v[start:end] } +func sliceJQValue(v JQValue, e, s any) any { + lv := v.JQValueSliceLen() + l, ok := lv.(int) + if !ok { + return lv + } + + var start, end int + if s != nil { + if i, ok := toInt(s); ok { + start = clampIndex(i, 0, l) + } else { + return &arrayIndexNotNumberError{s} + } + } + if e != nil { + if i, ok := toInt(e); ok { + end = clampIndex(i, start, l) + } else { + return &arrayIndexNotNumberError{e} + } + } else { + end = l + } + return v.JQValueSlice(start, end) +} + func clampIndex(i, min, max int) int { if i < 0 { i += max @@ -1135,7 +1246,7 @@ func funcFlatten(v any, args []any) any { func flatten(xs, vs []any, depth float64) []any { for _, v := range vs { - if vs, ok := v.([]any); ok && depth != 0 { + if vs, ok := toArray(v); ok && depth != 0 { xs = flatten(xs, vs, depth-1) } else { xs = append(xs, v) @@ -1158,7 +1269,12 @@ func (iter *rangeIter) Next() (any, bool) { } func funcRange(_ any, xs []any) any { - for _, x := range xs { + for i, x := range xs { + if jqv, ok := x.(JQValue); ok { + x = jqv.JQValueToGoJQ() + xs[i] = x + } + switch x.(type) { case int, float64, *big.Int: default: @@ -1169,7 +1285,7 @@ func funcRange(_ any, xs []any) any { } func funcMin(v any) any { - vs, ok := v.([]any) + vs, ok := toArray(v) if !ok { return &func0TypeError{"min", v} } @@ -1177,11 +1293,11 @@ func funcMin(v any) any { } func funcMinBy(v, x any) any { - vs, ok := v.([]any) + vs, ok := toArray(v) if !ok { return &func1TypeError{"min_by", v, x} } - xs, ok := x.([]any) + xs, ok := toArray(x) if !ok { return &func1TypeError{"min_by", v, x} } @@ -1192,7 +1308,7 @@ func funcMinBy(v, x any) any { } func funcMax(v any) any { - vs, ok := v.([]any) + vs, ok := toArray(v) if !ok { return &func0TypeError{"max", v} } @@ -1200,11 +1316,11 @@ func funcMax(v any) any { } func funcMaxBy(v, x any) any { - vs, ok := v.([]any) + vs, ok := toArray(v) if !ok { return &func1TypeError{"max_by", v, x} } - xs, ok := x.([]any) + xs, ok := toArray(x) if !ok { return &func1TypeError{"max_by", v, x} } @@ -1231,18 +1347,23 @@ type sortItem struct { value, key any } -func sortItems(name string, v, x any) ([]*sortItem, error) { - vs, ok := v.([]any) +func sortItems(name string, v, x any) ([]*sortItem, any) { + var ok bool + var vs []any + + vs, ok = toArray(v) if !ok { if strings.HasSuffix(name, "_by") { return nil, &func1TypeError{name, v, x} } return nil, &func0TypeError{name, v} } - xs, ok := x.([]any) + + xs, ok := toArray(x) if !ok { return nil, &func1TypeError{name, v, x} } + if len(vs) != len(xs) { return nil, &func1WrapError{name, v, x, &lengthMismatchError{}} } @@ -1324,12 +1445,16 @@ func funcJoin(v, x any) any { if len(vs) == 0 { return "" } - sep, ok := x.(string) + sep, ok := toString(x) if len(vs) > 1 && !ok { return &func1TypeError{"join", v, x} } ss := make([]string, len(vs)) for i, v := range vs { + if jqv, ok := v.(JQValue); ok { + v = jqv.JQValueToGoJQ() + } + switch v := v.(type) { case nil: case string: @@ -1493,7 +1618,7 @@ func funcSetpathWithAllocator(v any, args []any) any { } func setpath(v, p, n any, a allocator) any { - path, ok := p.([]any) + path, ok := toArray(p) if !ok { return &func1TypeError{"setpath", v, p} } @@ -1517,7 +1642,7 @@ func funcDelpathsWithAllocator(v any, args []any) any { } func delpaths(v, p any, a allocator) any { - paths, ok := p.([]any) + paths, ok := toArray(p) if !ok { return &func1TypeError{"delpaths", v, p} } @@ -1530,7 +1655,7 @@ func delpaths(v, p any, a allocator) any { var empty struct{} var err error for _, p := range paths { - path, ok := p.([]any) + path, ok := toArray(p) if !ok { return &func0TypeError{"delpaths", p} } @@ -1545,7 +1670,23 @@ func update(v any, path []any, n any, a allocator) (any, error) { if len(path) == 0 { return n, nil } - switch p := path[0].(type) { + + if jqv, ok := v.(JQValue); ok { + v = jqv.JQValueToGoJQ() + if err, ok := v.(error); ok { + return nil, err + } + } + + p0 := path[0] + if jqv, ok := p0.(JQValue); ok { + p0 = jqv.JQValueToGoJQ() + if err, ok := v.(error); ok { + return nil, err + } + } + + switch p := p0.(type) { case string: switch v := v.(type) { case nil: @@ -1738,14 +1879,14 @@ func deleteEmpty(v any) any { } func funcGetpath(v, p any) any { - keys, ok := p.([]any) + keys, ok := toArray(p) if !ok { return &func1TypeError{"getpath", v, p} } u := v for _, x := range keys { switch v.(type) { - case nil, []any, map[string]any: + case nil, []any, map[string]any, JQValue: v = funcIndex2(nil, v, x) if err, ok := v.(error); ok { return &func1WrapError{"getpath", u, p, err} @@ -1758,7 +1899,7 @@ func funcGetpath(v, p any) any { } func funcTranspose(v any) any { - vss, ok := v.([]any) + vss, ok := toArray(v) if !ok { return &func0TypeError{"transpose", v} } @@ -1767,7 +1908,7 @@ func funcTranspose(v any) any { } var l int for _, vs := range vss { - vs, ok := vs.([]any) + vs, ok := toArray(vs) if !ok { return &func0TypeError{"transpose", v} } @@ -1783,7 +1924,8 @@ func funcTranspose(v any) any { xs[i] = s } for i, vs := range vss { - for j, v := range vs.([]any) { + vs, _ := toArray(vs) + for j, v := range vs { wss[j][i] = v } } @@ -1791,7 +1933,7 @@ func funcTranspose(v any) any { } func funcBsearch(v, t any) any { - vs, ok := v.([]any) + vs, ok := toArray(v) if !ok { return &func1TypeError{"bsearch", v, t} } @@ -1833,7 +1975,7 @@ func epochToArray(v float64, loc *time.Location) []any { } func funcMktime(v any) any { - a, ok := v.([]any) + a, ok := toArray(v) if !ok { return &func0TypeError{"mktime", v} } @@ -1852,11 +1994,11 @@ func funcStrftime(v, x any) any { if w, ok := toFloat(v); ok { v = epochToArray(w, time.UTC) } - a, ok := v.([]any) + a, ok := toArray(v) if !ok { return &func1TypeError{"strftime", v, x} } - format, ok := x.(string) + format, ok := toString(x) if !ok { return &func1TypeError{"strftime", v, x} } @@ -1871,11 +2013,11 @@ func funcStrflocaltime(v, x any) any { if w, ok := toFloat(v); ok { v = epochToArray(w, time.Local) } - a, ok := v.([]any) + a, ok := toArray(v) if !ok { return &func1TypeError{"strflocaltime", v, x} } - format, ok := x.(string) + format, ok := toString(x) if !ok { return &func1TypeError{"strflocaltime", v, x} } @@ -1887,11 +2029,11 @@ func funcStrflocaltime(v, x any) any { } func funcStrptime(v, x any) any { - s, ok := v.(string) + s, ok := toString(v) if !ok { return &func1TypeError{"strptime", v, x} } - format, ok := x.(string) + format, ok := toString(x) if !ok { return &func1TypeError{"strptime", v, x} } @@ -1956,17 +2098,17 @@ func funcMatch(v, re, fs, testing any) any { } var flags string if fs != nil { - v, ok := fs.(string) + v, ok := toString(fs) if !ok { return &func2TypeError{name, v, re, fs} } flags = v } - s, ok := v.(string) + s, ok := toString(v) if !ok { return &func2TypeError{name, v, re, fs} } - restr, ok := re.(string) + restr, ok := toString(re) if !ok { return &func2TypeError{name, v, re, fs} } @@ -2087,6 +2229,50 @@ func funcHaltError(v any, args []any) any { return &exitCodeError{v, code, true} } +func toString(x any) (string, bool) { + switch x := x.(type) { + case string: + return x, true + case JQValue: + return toString(x.JQValueToGoJQ()) + default: + return "", false + } +} + +func toArray(x any) ([]any, bool) { + switch x := x.(type) { + case []any: + return x, true + case JQValue: + return toArray(x.JQValueToGoJQ()) + default: + return nil, false + } +} + +func toBoolean(x any) (bool, bool) { + switch x := x.(type) { + case bool: + return x, true + case JQValue: + return toBoolean(x.JQValueToGoJQ()) + default: + return false, false + } +} + +func isNull(x any) bool { + switch x := x.(type) { + case nil: + return true + case JQValue: + return isNull(x.JQValueToGoJQ()) + default: + return false + } +} + func toInt(x any) (int, bool) { switch x := x.(type) { case int: @@ -2103,6 +2289,8 @@ func toInt(x any) (int, bool) { return math.MaxInt, true } return math.MinInt, true + case JQValue: + return toInt(x.JQValueToGoJQ()) default: return 0, false } @@ -2126,6 +2314,8 @@ func toFloat(x any) (float64, bool) { return x, true case *big.Int: return bigToFloat(x), true + case JQValue: + return toFloat(x.JQValueToGoJQ()) default: return 0.0, false } diff --git a/gojq.go b/gojq.go index e078c809..fe807b87 100644 --- a/gojq.go +++ b/gojq.go @@ -3,3 +3,67 @@ // // [Usage as a library]: https://github.com/itchyny/gojq#usage-as-a-library package gojq + +// TODO: use in TypeOf? +const ( + JQTypeArray = "array" + JQTypeBoolean = "boolean" + JQTypeNull = "null" + JQTypeNumber = "number" + JQTypeObject = "object" + JQTypeString = "string" +) + +// JQValue represents something that can be a jq value +// All functions with return type any can return error on error +// array = []any +// boolean = bool +// null = nil +// number = int | float64 | *big.Int +// object = map[string]any +// string = string +// value = number | boolean | string | array | object | null +type JQValue interface { + // JQValueLength is length of value, ex: value | length + JQValueLength() any // int + + // JQValueSliceLen slice length + JQValueSliceLen() any // int + + // JQValueIndex look up index for value, ex: value[index] + // index -1 outside after slice, -2 outside before slice + JQValueIndex(index int) any // value + + // JQValueSlice slices value, ex: value[start:end] + // start and end indexes are translated and clamped using JQValueSliceLen + JQValueSlice(start int, end int) any // []any + + // JQValueKey look up key value of value: ex: value[name] + JQValueKey(name string) any // value + + // JQValueEach each of value, ex: value[] + JQValueEach() any // []PathValue + + // JQValueKeys keys for value, ex: value | keys + JQValueKeys() any // []string + + // JQValueHas check if value has key, ex: value | has(key) + JQValueHas(key any) any // bool + + // JQValueType type of value, ex: value | type + JQValueType() string // a JQType* constant + + // JQValueToNumber is value represented as a number, ex: value | tonumber + JQValueToNumber() any // number + + // JQValueToString is value represented as a string, ex: value | tostring + JQValueToString() any // string + + // JQValue value represented as a gojq compatible value + JQValueToGoJQ() any // value +} + +// PathValue is a part of a jq path expression +type PathValue struct { + Path, Value any +} diff --git a/normalize.go b/normalize.go index f9a6cde1..4131ffa0 100644 --- a/normalize.go +++ b/normalize.go @@ -7,6 +7,14 @@ import ( "strings" ) +func ValidNumber(s string) bool { + return newLexer(s).validNumber() +} + +func NormalizeNumber(v json.Number) any { + return normalizeNumber(v) +} + func normalizeNumber(v json.Number) any { if i, err := v.Int64(); err == nil && math.MinInt <= i && i <= math.MaxInt { return int(i) @@ -25,6 +33,10 @@ func normalizeNumber(v json.Number) any { return math.Inf(1) } +func NormalizeNumbers(v any) any { + return normalizeNumbers(v) +} + func normalizeNumbers(v any) any { switch v := v.(type) { case json.Number: diff --git a/operator.go b/operator.go index 111bc0c3..9bc19fb2 100644 --- a/operator.go +++ b/operator.go @@ -317,6 +317,14 @@ func binopTypeSwitch( callbackArrays func(_, _ []any) any, callbackMaps func(_, _ map[string]any) any, fallback func(_, _ any) any) any { + + if lj, ok := l.(JQValue); ok { + l = lj.JQValueToGoJQ() + } + if rj, ok := r.(JQValue); ok { + r = rj.JQValueToGoJQ() + } + switch l := l.(type) { case int: switch r := r.(type) { @@ -385,6 +393,8 @@ func funcOpPlus(v any) any { return v case *big.Int: return v + case JQValue: + return funcOpPlus(v.JQValueToGoJQ()) default: return &unaryTypeError{"plus", v} } @@ -398,6 +408,8 @@ func funcOpNegate(v any) any { return -v case *big.Int: return new(big.Int).Neg(v) + case JQValue: + return funcOpNegate(v.JQValueToGoJQ()) default: return &unaryTypeError{"negate", v} } @@ -450,6 +462,13 @@ func funcOpAdd(_, l, r any) any { if r == nil { return l } + + if isNull(l) { + return r + } else if isNull(r) { + return l + } + return &binopTypeError{"add", l, r} }, ) @@ -625,7 +644,9 @@ func funcOpMod(_, l, r any) any { } func funcOpAlt(_, l, r any) any { - if l == nil || l == false { + if isNull(l) { + return r + } else if lb, ok := toBoolean(l); ok && !lb { return r } return l diff --git a/preview_test.go b/preview_test.go index 5b6c878f..d1d81839 100644 --- a/preview_test.go +++ b/preview_test.go @@ -6,7 +6,7 @@ import ( "math/big" "testing" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) func TestPreview(t *testing.T) { diff --git a/type.go b/type.go index bb388e20..a8a6f29f 100644 --- a/type.go +++ b/type.go @@ -8,9 +8,10 @@ import ( // TypeOf returns the jq-flavored type name of v. // // This method is used by built-in type/0 function, and accepts only limited -// types (nil, bool, int, float64, *big.Int, string, []any, and map[string]any). +// types (nil, bool, int, float64, *big.Int, string, []any, +// and map[string]any). func TypeOf(v any) string { - switch v.(type) { + switch v := v.(type) { case nil: return "null" case bool: @@ -23,6 +24,8 @@ func TypeOf(v any) string { return "array" case map[string]any: return "object" + case JQValue: + return v.JQValueType() default: panic(fmt.Sprintf("invalid type: %[1]T (%[1]v)", v)) } diff --git a/type_test.go b/type_test.go index ef917a02..0ee14660 100644 --- a/type_test.go +++ b/type_test.go @@ -6,7 +6,7 @@ import ( "math/big" "testing" - "github.com/itchyny/gojq" + "github.com/wader/gojq" ) func TestTypeOf(t *testing.T) { From 9848f23b4a047d9fa52b82457faab783b66e5334 Mon Sep 17 00:00:00 2001 From: Matt Dale <9760375+matthewdale@users.noreply.github.com> Date: Sun, 4 Jun 2023 14:15:02 -0700 Subject: [PATCH 9/9] support decimal numbers in Query.Run --- compare.go | 5 +++++ compare_test.go | 11 +++++++++++ encoder.go | 4 ++++ func.go | 2 ++ go.mod | 1 + go.sum | 2 ++ operator.go | 41 +++++++++++++++++++++++++++++++++++++++++ preview_test.go | 5 +++++ query_test.go | 3 ++- type.go | 4 +++- type_test.go | 2 ++ 11 files changed, 78 insertions(+), 2 deletions(-) diff --git a/compare.go b/compare.go index e70c1fbb..58bb7ffb 100644 --- a/compare.go +++ b/compare.go @@ -3,6 +3,8 @@ package gojq import ( "math" "math/big" + + "github.com/shopspring/decimal" ) // Compare l and r, and returns jq-flavored comparison value. @@ -25,6 +27,9 @@ func compare(l, r any) int { return 1 } }, + func(l, r decimal.Decimal) any { + return l.Cmp(r) + }, func(l, r *big.Int) any { return l.Cmp(r) }, diff --git a/compare_test.go b/compare_test.go index 3bdce2e3..7fd7d525 100644 --- a/compare_test.go +++ b/compare_test.go @@ -6,6 +6,7 @@ import ( "math/big" "testing" + "github.com/shopspring/decimal" "github.com/wader/gojq" ) @@ -35,6 +36,16 @@ func TestCompare(t *testing.T) { {1.00, 1.01, -1}, {1.01, 1.00, 1}, {1.01, 1.01, 0}, + {decimal.New(1, 0), decimal.New(1, 0), 0}, + {decimal.NewFromFloat(1.00), decimal.NewFromFloat(1.01), -1}, + {decimal.NewFromFloat(1.01), decimal.NewFromFloat(1.00), 1}, + {1, decimal.New(1, 0), 0}, + {decimal.New(1, 0), 1, 0}, + {1.00, decimal.NewFromFloat(1.01), -1}, + {decimal.NewFromFloat(1.01), 1.00, 1}, + {1.01, decimal.NewFromFloat(1.00), 1}, + {big.NewInt(1), decimal.NewFromFloat(1.00), 0}, + {decimal.NewFromFloat(1.01), big.NewInt(1), 1}, {1, big.NewInt(0), 1}, {big.NewInt(0), 1, -1}, {0, big.NewInt(0), 0}, diff --git a/encoder.go b/encoder.go index e3c4f0f2..f02617a7 100644 --- a/encoder.go +++ b/encoder.go @@ -10,6 +10,8 @@ import ( "strconv" "strings" "unicode/utf8" + + "github.com/shopspring/decimal" ) // Marshal returns the jq-flavored JSON encoding of v. @@ -60,6 +62,8 @@ func (e *encoder) encode(v any) { e.w.Write(strconv.AppendInt(e.buf[:0], int64(v), 10)) case float64: e.encodeFloat64(v) + case decimal.Decimal: + e.w.WriteString(v.String()) case *big.Int: e.w.Write(v.Append(e.buf[:0], 10)) case string: diff --git a/func.go b/func.go index 3635ff53..1dbf56fd 100644 --- a/func.go +++ b/func.go @@ -18,6 +18,7 @@ import ( "unicode/utf8" "github.com/itchyny/timefmt-go" + "github.com/shopspring/decimal" ) //go:generate go run -modfile=go.dev.mod _tools/gen_builtin.go -i builtin.jq -o builtin.go @@ -602,6 +603,7 @@ func funcContains(v, x any) any { return binopTypeSwitch(v, x, func(l, r int) any { return l == r }, func(l, r float64) any { return l == r }, + func(l, r decimal.Decimal) any { return l.Equal(r) }, func(l, r *big.Int) any { return l.Cmp(r) == 0 }, func(l, r string) any { return strings.Contains(l, r) }, func(l, r []any) any { diff --git a/go.mod b/go.mod index 4ce518d1..5af1140b 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/itchyny/timefmt-go v0.1.5 github.com/mattn/go-isatty v0.0.17 github.com/mattn/go-runewidth v0.0.14 + github.com/shopspring/decimal v1.3.1 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 051df647..09e28188 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/operator.go b/operator.go index 9bc19fb2..843b8b8a 100644 --- a/operator.go +++ b/operator.go @@ -6,6 +6,8 @@ import ( "math" "math/big" "strings" + + "github.com/shopspring/decimal" ) // Operator ... @@ -312,6 +314,7 @@ func binopTypeSwitch( l, r any, callbackInts func(_, _ int) any, callbackFloats func(_, _ float64) any, + callbackDecimals func(_, _ decimal.Decimal) any, callbackBigInts func(_, _ *big.Int) any, callbackStrings func(_, _ string) any, callbackArrays func(_, _ []any) any, @@ -332,6 +335,8 @@ func binopTypeSwitch( return callbackInts(l, r) case float64: return callbackFloats(float64(l), r) + case decimal.Decimal: + return callbackDecimals(decimal.NewFromInt(int64(l)), r) case *big.Int: return callbackBigInts(big.NewInt(int64(l)), r) default: @@ -343,17 +348,34 @@ func binopTypeSwitch( return callbackFloats(l, float64(r)) case float64: return callbackFloats(l, r) + case decimal.Decimal: + return callbackDecimals(decimal.NewFromFloat(l), r) case *big.Int: return callbackFloats(l, bigToFloat(r)) default: return fallback(l, r) } + case decimal.Decimal: + switch r := r.(type) { + case int: + return callbackDecimals(l, decimal.NewFromInt(int64(r))) + case float64: + return callbackDecimals(l, decimal.NewFromFloat(r)) + case decimal.Decimal: + return callbackDecimals(l, r) + case *big.Int: + return callbackDecimals(l, decimal.NewFromBigInt(r, 0)) + default: + return fallback(l, r) + } case *big.Int: switch r := r.(type) { case int: return callbackBigInts(l, big.NewInt(int64(r))) case float64: return callbackFloats(bigToFloat(l), r) + case decimal.Decimal: + return callbackDecimals(decimal.NewFromBigInt(l, 0), r) case *big.Int: return callbackBigInts(l, r) default: @@ -425,6 +447,7 @@ func funcOpAdd(_, l, r any) any { return x.Add(x, y) }, func(l, r float64) any { return l + r }, + func(l, r decimal.Decimal) any { return l.Add(r) }, func(l, r *big.Int) any { return new(big.Int).Add(l, r) }, func(l, r string) any { return l + r }, func(l, r []any) any { @@ -484,6 +507,7 @@ func funcOpSub(_, l, r any) any { return x.Sub(x, y) }, func(l, r float64) any { return l - r }, + func(l, r decimal.Decimal) any { return l.Sub(r) }, func(l, r *big.Int) any { return new(big.Int).Sub(l, r) }, func(l, r string) any { return &binopTypeError{"subtract", l, r} }, func(l, r []any) any { @@ -514,6 +538,7 @@ func funcOpMul(_, l, r any) any { return x.Mul(x, y) }, func(l, r float64) any { return l * r }, + func(l, r decimal.Decimal) any { return l.Mul(r) }, func(l, r *big.Int) any { return new(big.Int).Mul(l, r) }, func(l, r string) any { return &binopTypeError{"multiply", l, r} }, func(l, r []any) any { return &binopTypeError{"multiply", l, r} }, @@ -585,6 +610,15 @@ func funcOpDiv(_, l, r any) any { } return l / r }, + func(l, r decimal.Decimal) any { + if r.IsZero() { + if l.IsZero() { + return math.NaN() + } + return &zeroDivisionError{l, r} + } + return l.Div(r) + }, func(l, r *big.Int) any { if r.Sign() == 0 { if l.Sign() == 0 { @@ -630,6 +664,13 @@ func funcOpMod(_, l, r any) any { } return floatToInt(l) % ri }, + func(l, r decimal.Decimal) any { + ri := r.BigInt() + if ri.Sign() == 0 { + return &zeroModuloError{l, r} + } + return new(big.Int).Rem(l.BigInt(), ri) + }, func(l, r *big.Int) any { if r.Sign() == 0 { return &zeroModuloError{l, r} diff --git a/preview_test.go b/preview_test.go index d1d81839..4ee261f2 100644 --- a/preview_test.go +++ b/preview_test.go @@ -6,6 +6,7 @@ import ( "math/big" "testing" + "github.com/shopspring/decimal" "github.com/wader/gojq" ) @@ -46,6 +47,10 @@ func TestPreview(t *testing.T) { math.Inf(-1), "-1.7976931348623157e+308", }, + { + decimal.New(1234567890123456789, -18), + "1.234567890123456789", + }, { big.NewInt(9223372036854775807), "9223372036854775807", diff --git a/query_test.go b/query_test.go index e2065d21..956d795d 100644 --- a/query_test.go +++ b/query_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "github.com/shopspring/decimal" "github.com/wader/gojq" ) @@ -215,7 +216,7 @@ func TestQueryRun_NumericTypes(t *testing.T) { int64(math.MaxInt64), int64(math.MinInt64), uint64(math.MaxUint64), uint32(math.MaxUint32), new(big.Int).SetUint64(math.MaxUint64), new(big.Int).SetUint64(math.MaxUint32), json.Number(fmt.Sprint(uint64(math.MaxInt64))), json.Number(fmt.Sprint(uint64(math.MaxInt32))), - float64(1.0), float32(1.0), + float64(1.0), float32(1.0), decimal.New(1, 0), }) for { v, ok := iter.Next() diff --git a/type.go b/type.go index a8a6f29f..784a5d01 100644 --- a/type.go +++ b/type.go @@ -3,6 +3,8 @@ package gojq import ( "fmt" "math/big" + + "github.com/shopspring/decimal" ) // TypeOf returns the jq-flavored type name of v. @@ -16,7 +18,7 @@ func TypeOf(v any) string { return "null" case bool: return "boolean" - case int, float64, *big.Int: + case int, float64, *big.Int, decimal.Decimal: return "number" case string: return "string" diff --git a/type_test.go b/type_test.go index 0ee14660..50dc4897 100644 --- a/type_test.go +++ b/type_test.go @@ -6,6 +6,7 @@ import ( "math/big" "testing" + "github.com/shopspring/decimal" "github.com/wader/gojq" ) @@ -19,6 +20,7 @@ func TestTypeOf(t *testing.T) { {true, "boolean"}, {0, "number"}, {3.14, "number"}, + {decimal.New(1, 0), "number"}, {math.NaN(), "number"}, {math.Inf(1), "number"}, {math.Inf(-1), "number"},