Skip to content

Commit

Permalink
fq: Generate decode alises code
Browse files Browse the repository at this point in the history
  • Loading branch information
@wader
wader committed Sep 12, 2021
1 parent 834f4a5 commit 8cb380e
Show file tree
Hide file tree
Showing 4 changed files with 96 additions and 49 deletions.
87 changes: 39 additions & 48 deletions pkg/interp/funcs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ import (
"net/url"
"time"

"github.com/wader/fq/format"
"github.com/wader/fq/format/registry"
"github.com/wader/fq/internal/aheadreadseeker"
"github.com/wader/fq/internal/ctxreadseeker"
"github.com/wader/fq/internal/gojqextra"
Expand All @@ -34,7 +32,7 @@ import (
)

// TODO: make it nicer somehow? generate generators? remove from struct?
func (i *Interp) makeFunctions(registry *registry.Registry) []Function {
func (i *Interp) makeFunctions() []Function {
fs := []Function{
{[]string{"readline"}, 0, 2, i.readline, nil},
{[]string{"eval"}, 1, 2, nil, i.eval},
Expand All @@ -54,10 +52,10 @@ func (i *Interp) makeFunctions(registry *registry.Registry) []Function {
{[]string{"history"}, 0, 0, i.history, nil},

{[]string{"open"}, 0, 0, i._open, nil},
{[]string{"decode"}, 0, 1, i.makeDecodeFn(registry, registry.MustGroup(format.PROBE)), nil},
{[]string{"_decode"}, 2, 2, i._decode, nil},

{[]string{"format"}, 0, 0, i.format, nil},
{[]string{"_display"}, 1, 1, nil, i.display},
{[]string{"_display"}, 1, 1, nil, i._display},
{[]string{"preview", "p"}, 0, 1, nil, i.preview},
{[]string{"hexdump", "hd", "h"}, 0, 1, nil, i.hexdump},

Expand Down Expand Up @@ -98,9 +96,6 @@ func (i *Interp) makeFunctions(registry *registry.Registry) []Function {

{[]string{"find"}, 1, 1, nil, i.find},
}
for name, f := range i.registry.Groups {
fs = append(fs, Function{[]string{name}, 0, 0, i.makeDecodeFn(registry, f), nil})
}

return fs
}
Expand Down Expand Up @@ -537,56 +532,52 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
}
}

func (i *Interp) makeDecodeFn(registry *registry.Registry, decodeFormats []*decode.Format) func(c interface{}, a []interface{}) interface{} {
return func(c interface{}, a []interface{}) interface{} {
filename := "unnamed"

// TODO: progress hack
// would be nice to move progress code into decode but it might be
// tricky to keep track of absolute positions in the underlaying readers
// when it uses BitBuf slices, maybe only in Pos()?
if bbf, ok := c.(*bitBufFile); ok {
if bbf.decodeDoneFn != nil {
defer bbf.decodeDoneFn()
}
filename = bbf.filename
}
func (i *Interp) _decode(c interface{}, a []interface{}) interface{} {
filename := "unnamed"

bb, err := toBuffer(c)
if err != nil {
return err
// TODO: progress hack
// would be nice to move progress code into decode but it might be
// tricky to keep track of absolute positions in the underlaying readers
// when it uses BitBuf slices, maybe only in Pos()?
if bbf, ok := c.(*bitBufFile); ok {
if bbf.decodeDoneFn != nil {
defer bbf.decodeDoneFn()
}
filename = bbf.filename
}

opts := map[string]interface{}{}
bb, err := toBuffer(c)
if err != nil {
return err
}

if len(a) >= 1 {
formatName, err := toString(a[0])
if err != nil {
return fmt.Errorf("%s: %w", formatName, err)
}
decodeFormats, err = registry.Group(formatName)
if err != nil {
return fmt.Errorf("%s: %w", formatName, err)
}
}
opts := map[string]interface{}{}

dv, _, err := decode.Decode("", filename, bb, decodeFormats, decode.DecodeOptions{FormatOptions: opts})
if dv == nil {
var decodeFormatsErr decode.DecodeFormatsError
if errors.As(err, &decodeFormatsErr) {
var vs []interface{}
for _, fe := range decodeFormatsErr.Errs {
vs = append(vs, fe.Value())
}
formatName, err := toString(a[0])
if err != nil {
return fmt.Errorf("%s: %w", formatName, err)
}
decodeFormats, err := i.registry.Group(formatName)
if err != nil {
return fmt.Errorf("%s: %w", formatName, err)
}

return valueError{vs}
dv, _, err := decode.Decode("", filename, bb, decodeFormats, decode.DecodeOptions{FormatOptions: opts})
if dv == nil {
var decodeFormatsErr decode.DecodeFormatsError
if errors.As(err, &decodeFormatsErr) {
var vs []interface{}
for _, fe := range decodeFormatsErr.Errs {
vs = append(vs, fe.Value())
}

return valueError{err}
return valueError{vs}
}

return makeDecodeValue(dv)
return valueError{err}
}

return makeDecodeValue(dv)
}

func (i *Interp) format(c interface{}, a []interface{}) interface{} {
Expand All @@ -601,7 +592,7 @@ func (i *Interp) format(c interface{}, a []interface{}) interface{} {
return f
}

func (i *Interp) display(c interface{}, a []interface{}) gojq.Iter {
func (i *Interp) _display(c interface{}, a []interface{}) gojq.Iter {
opts, err := i.Options(a...)
if err != nil {
return gojq.NewIter(err)
Expand Down
14 changes: 13 additions & 1 deletion pkg/interp/interp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -570,7 +570,7 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri
}

var compilerOpts []gojq.CompilerOption
for _, f := range ni.makeFunctions(ni.registry) {
for _, f := range ni.makeFunctions() {
for _, n := range f.Names {
if f.IterFn != nil {
compilerOpts = append(compilerOpts,
Expand Down Expand Up @@ -621,6 +621,18 @@ func (i *Interp) Eval(ctx context.Context, mode RunMode, c interface{}, src stri
fmt.Fprintf(sb, "include \"@format/%s\";\n", f.Name)
}
return bytes.NewReader(sb.Bytes()), nil
} else if filename == "decode.jq" {
sb := &bytes.Buffer{}
fmt.Fprintf(sb, "def decode($name; $opts): _decode($name; $opts);\n")
fmt.Fprintf(sb, "def decode($name): _decode($name; {});\n")
fmt.Fprintf(sb, "def decode: _decode(\"probe\"; {});\n")
for name := range i.registry.Groups {
fmt.Fprintf(sb, ""+
"def %[1]s($opts): _decode(%[1]q; $opts);\n"+
"def %[1]s: _decode(%[1]q; {});\n",
name)
}
return bytes.NewReader(sb.Bytes()), nil
} else {
formatName := strings.TrimRight(filename, ".jq")
for _, f := range allFormats {
Expand Down
1 change: 1 addition & 0 deletions pkg/interp/interp.jq
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ include "args";
include "query";

# will include all per format specific function etc
include "@format/decode";
include "@format/all";

# optional user init
Expand Down
43 changes: 43 additions & 0 deletions pkg/interp/testdata/decode.fqtest
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/test.mp3:
$ fq -i -d mp3 . /test.mp3
mp3> decode
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f| |.: {} unnamed (mp3)
0x000|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..| headers: [1]
* |until 0x2c.7 (45) | |
0x020| ff fb 40| ..@| frames: [3]
0x030|c0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x283.7 (end) (599) | |
| | | footers: [0]
mp3> decode("mp3")
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f| |.: {} unnamed (mp3)
0x000|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..| headers: [1]
* |until 0x2c.7 (45) | |
0x020| ff fb 40| ..@| frames: [3]
0x030|c0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x283.7 (end) (599) | |
| | | footers: [0]
mp3> decode("mp3"; {})
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f| |.: {} unnamed (mp3)
0x000|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..| headers: [1]
* |until 0x2c.7 (45) | |
0x020| ff fb 40| ..@| frames: [3]
0x030|c0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x283.7 (end) (599) | |
| | | footers: [0]
mp3> probe
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f| |.: {} unnamed (mp3)
0x000|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..| headers: [1]
* |until 0x2c.7 (45) | |
0x020| ff fb 40| ..@| frames: [3]
0x030|c0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x283.7 (end) (599) | |
| | | footers: [0]
mp3> probe({})
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f| |.: {} unnamed (mp3)
0x000|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..| headers: [1]
* |until 0x2c.7 (45) | |
0x020| ff fb 40| ..@| frames: [3]
0x030|c0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x283.7 (end) (599) | |
| | | footers: [0]
mp3> ^D

0 comments on commit 8cb380e

Please sign in to comment.