Skip to content

Commit

Permalink
decode,interp: Don't shadow _key and error on missing _key
Browse files Browse the repository at this point in the history 
For fromjson and other "value" decode values fq make then behave both like
a normal jq value and decode value. This is to make tobytes, format etc work.
Before all _* would be treated as special keys. Now they are first looked up in
the wrapped value and then as decode values.

Also now ._key that don't exist reutrn null instead of throw error.

$ fq -n '`{"_format": 123}` | fromjson | ._format'
Now:
123
Before:
"json"

$ fq -n '`{}` | fromjson | ._missing'
Now:
null
Before
error
  • Loading branch information
@wader
wader committed Sep 3, 2023
1 parent d97876c commit c0352f2
Show file tree
Hide file tree
Showing 10 changed files with 158 additions and 102 deletions.
130 changes: 79 additions & 51 deletions pkg/interp/decode.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"io"
"math/big"
"reflect"
"strings"
"time"

"github.com/mitchellh/copystructure"
Expand All @@ -27,14 +26,6 @@ func init() {
RegisterFunc2("_decode", (*Interp)._decode)
}

type expectedExtkeyError struct {
Key string
}

func (err expectedExtkeyError) Error() string {
return "expected a extkey but got: " + err.Key
}

// TODO: redo/rename
// used by _isDecodeValue
type DecodeValue interface {
Expand Down Expand Up @@ -294,21 +285,19 @@ func (i *Interp) _decode(c any, format string, opts decodeOpts) any {
return makeDecodeValueOut(dv, decodeValueValue, formatOutMap)
}

func valueKey(name string, a, b func(name string) any) any {
if strings.HasPrefix(name, "_") {
return a(name)
func valueOrFallbackKey(name string, baseKey func(name string) any, valueHas func(key any) any, valueKey func(name string) any) any {
v := valueHas(name)
if b, ok := v.(bool); ok && b {
return valueKey(name)
}
return b(name)
return baseKey(name)
}
func valueHas(key any, a func(name string) any, b func(key any) any) any {
stringKey, ok := key.(string)
if ok && strings.HasPrefix(stringKey, "_") {
if err, ok := a(stringKey).(error); ok {
return err
}
return true
func valueOrFallbackHas(key any, baseHas func(key any) any, valueHas func(key any) any) any {
v := valueHas(key)
if b, ok := v.(bool); ok && !b {
return baseHas(key)
}
return b(key)
return v
}

// TODO: make more efficient somehow? shallow values but might be hard
Expand Down Expand Up @@ -461,37 +450,59 @@ func (dvb decodeValueBase) ToBinary() (Binary, error) {
}
func (decodeValueBase) ExtType() string { return "decode_value" }
func (dvb decodeValueBase) ExtKeys() []string {
kv := []string{
return []string{
"_actual",
"_bits",
"_buffer_root",
"_bytes",
"_description",
"_error",
"_format_root",
"_format",
"_gap",
"_index",
"_len",
"_name",
"_out",
"_parent",
"_path",
"_root",
"_start",
"_stop",
"_sym",
}
}

if _, ok := dvb.dv.V.(*decode.Compound); ok {
kv = append(kv,
"_error",
"_format",
"_out",
)
func (dvb decodeValueBase) JQValueHas(key any) any {
name, ok := key.(string)
if !ok {
return false
}

if dvb.dv.Index != -1 {
kv = append(kv, "_index")
}
switch name {
case "_actual",
"_bits",
"_buffer_root",
"_bytes",
"_description",
"_error",
"_format_root",
"_format",
"_gap",
"_index",
"_len",
"_name",
"_out",
"_parent",
"_path",
"_root",
"_start",
"_stop",
"_sym":
return true
}

return kv
return false
}

func (dvb decodeValueBase) JQValueKey(name string) any {
Expand Down Expand Up @@ -591,7 +602,7 @@ func (dvb decodeValueBase) JQValueKey(name string) any {
}
}

return expectedExtkeyError{Key: name}
return nil
}

var _ DecodeValue = decodeValue{}
Expand All @@ -603,10 +614,10 @@ type decodeValue struct {
}

func (v decodeValue) JQValueKey(name string) any {
return valueKey(name, v.decodeValueBase.JQValueKey, v.JQValue.JQValueKey)
return valueOrFallbackKey(name, v.decodeValueBase.JQValueKey, v.JQValue.JQValueHas, v.JQValue.JQValueKey)
}
func (v decodeValue) JQValueHas(key any) any {
return valueHas(key, v.decodeValueBase.JQValueKey, v.JQValue.JQValueHas)
return valueOrFallbackHas(key, v.decodeValueBase.JQValueHas, v.JQValue.JQValueHas)
}
func (v decodeValue) JQValueToGoJQEx(optsFn func() (*Options, error)) any {
if !v.isRaw {
Expand Down Expand Up @@ -641,7 +652,7 @@ func NewArrayDecodeValue(dv *decode.Value, out any, c *decode.Compound) ArrayDec
}

func (v ArrayDecodeValue) JQValueKey(name string) any {
return valueKey(name, v.decodeValueBase.JQValueKey, v.Base.JQValueKey)
return valueOrFallbackKey(name, v.decodeValueBase.JQValueKey, v.Base.JQValueHas, v.Base.JQValueKey)
}
func (v ArrayDecodeValue) JQValueSliceLen() any { return len(v.Compound.Children) }
func (v ArrayDecodeValue) JQValueLength() any { return len(v.Compound.Children) }
Expand Down Expand Up @@ -674,9 +685,9 @@ func (v ArrayDecodeValue) JQValueKeys() any {
return vs
}
func (v ArrayDecodeValue) JQValueHas(key any) any {
return valueHas(
return valueOrFallbackHas(
key,
v.decodeValueBase.JQValueKey,
v.decodeValueBase.JQValueHas,
func(key any) any {
intKey, ok := key.(int)
if !ok {
Expand Down Expand Up @@ -730,16 +741,31 @@ func NewStructDecodeValue(dv *decode.Value, out any, c *decode.Compound) StructD
func (v StructDecodeValue) JQValueLength() any { return len(v.Compound.Children) }
func (v StructDecodeValue) JQValueSliceLen() any { return len(v.Compound.Children) }
func (v StructDecodeValue) JQValueKey(name string) any {
if strings.HasPrefix(name, "_") {
return v.decodeValueBase.JQValueKey(name)
}
if v.Compound.ByName != nil {
if f, ok := v.Compound.ByName[name]; ok {
return makeDecodeValue(f, decodeValueValue)
}
}
return valueOrFallbackKey(
name,
v.decodeValueBase.JQValueKey,
func(key any) any {
stringKey, ok := key.(string)
if !ok {
return false
}
if v.Compound.ByName != nil {
if _, ok := v.Compound.ByName[stringKey]; ok {
return true
}
}
return false
},
func(name string) any {
if v.Compound.ByName != nil {
if f, ok := v.Compound.ByName[name]; ok {
return makeDecodeValue(f, decodeValueValue)
}
}

return nil
return nil
},
)
}
func (v StructDecodeValue) JQValueEach() any {
props := make([]gojq.PathValue, len(v.Compound.Children))
Expand All @@ -756,19 +782,21 @@ func (v StructDecodeValue) JQValueKeys() any {
return vs
}
func (v StructDecodeValue) JQValueHas(key any) any {
return valueHas(
return valueOrFallbackHas(
key,
v.decodeValueBase.JQValueKey,
v.decodeValueBase.JQValueHas,
func(key any) any {
stringKey, ok := key.(string)
if !ok {
return gojqex.HasKeyTypeError{L: gojq.JQTypeObject, R: fmt.Sprintf("%v", key)}
}
for _, f := range v.Compound.Children {
if f.Name == stringKey {

if v.Compound.ByName != nil {
if _, ok := v.Compound.ByName[stringKey]; ok {
return true
}
}

return false
},
)
Expand Down
1 change: 1 addition & 0 deletions pkg/interp/testdata/completion.fqtest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ _error
_format
_format_root
_gap
_index
_len
_name
_out
Expand Down
76 changes: 38 additions & 38 deletions pkg/interp/testdata/value.fqtest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,77 +127,77 @@ true
"_format"
true
"_abc"
"expected a extkey but got: _abc"
false
mp3> .headers as $c | ("_start", "_stop", "_len", "_name", "_root", "_buffer_root", "_format_root", "_parent", "_sym", "_description", "_path", "_bits", "_bytes", "_error", "_gap", "_format", "_abc") as $n | $n, try ($c | has($n)) catch .
"_start"
true
"cannot check whether array has a key: _start"
"_stop"
true
"cannot check whether array has a key: _stop"
"_len"
true
"cannot check whether array has a key: _len"
"_name"
true
"cannot check whether array has a key: _name"
"_root"
true
"cannot check whether array has a key: _root"
"_buffer_root"
true
"cannot check whether array has a key: _buffer_root"
"_format_root"
true
"cannot check whether array has a key: _format_root"
"_parent"
true
"cannot check whether array has a key: _parent"
"_sym"
true
"cannot check whether array has a key: _sym"
"_description"
true
"cannot check whether array has a key: _description"
"_path"
true
"cannot check whether array has a key: _path"
"_bits"
true
"cannot check whether array has a key: _bits"
"_bytes"
true
"cannot check whether array has a key: _bytes"
"_error"
true
"cannot check whether array has a key: _error"
"_gap"
true
"cannot check whether array has a key: _gap"
"_format"
true
"cannot check whether array has a key: _format"
"_abc"
"expected a extkey but got: _abc"
"cannot check whether array has a key: _abc"
mp3> .headers[0].header.magic as $c | ("_start", "_stop", "_len", "_name", "_root", "_buffer_root", "_format_root", "_parent", "_sym", "_description", "_path", "_bits", "_bytes", "_error", "_gap", "_format", "_abc") as $n | $n, try ($c | has($n)) catch .
"_start"
true
"has cannot be applied to: string"
"_stop"
true
"has cannot be applied to: string"
"_len"
true
"has cannot be applied to: string"
"_name"
true
"has cannot be applied to: string"
"_root"
true
"has cannot be applied to: string"
"_buffer_root"
true
"has cannot be applied to: string"
"_format_root"
true
"has cannot be applied to: string"
"_parent"
true
"has cannot be applied to: string"
"_sym"
true
"has cannot be applied to: string"
"_description"
true
"has cannot be applied to: string"
"_path"
true
"has cannot be applied to: string"
"_bits"
true
"has cannot be applied to: string"
"_bytes"
true
"has cannot be applied to: string"
"_error"
true
"has cannot be applied to: string"
"_gap"
true
"has cannot be applied to: string"
"_format"
true
"has cannot be applied to: string"
"_abc"
"expected a extkey but got: _abc"
"has cannot be applied to: string"
mp3> ^D
$ fq -d mp3 -i . test.mp3
mp3> ._start
Expand Down Expand Up @@ -255,13 +255,13 @@ false
mp3> ._format
"mp3"
mp3> ._abc
error: expected a extkey but got: _abc
null
mp3> fgrep("toc") | arrays[10]._index
10
mp3> .headers._index
error: expected a extkey but got: _index
null
mp3> ._index
error: expected a extkey but got: _index
null
mp3> .headers[0].header.version as $r | {a:12} | tojson({indent: $r}) | println
{
"a": 12
Expand Down
6 changes: 4 additions & 2 deletions pkg/interp/testdata/value_array.fqtest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,9 @@ mp3> .headers[-1000:2000] | ., type, length?
"array"
1
mp3> .headers["test"] | ., type, length?
error: expected an object with key "test" but got: array
null
"null"
0
mp3> [.headers[]] | type, length?
"array"
1
Expand Down Expand Up @@ -522,7 +524,7 @@ mp3> .headers[0] = 1
]
}
mp3> .headers.a |= empty
error: expected an object with key "a" but got: array
error: delpaths([["headers","a"]]) cannot be applied to {"footers":[],"frames":[{" ...: expected an object but got: array ([{"frames":[{"flags":{"co ...])
mp3> .headers[0] |= empty
{
"footers": [],
Expand Down

0 comments on commit c0352f2

Please sign in to comment.