interp: Add paste function to allow pasting text into REPL etc

Also refactor readline and eval args into option struct and partinally start
addressing some side effects during completion.

doc/TODO.md

@@ -13,6 +13,7 @@
 - Rework cli/repl user interrupt (context cancel via ctrl-c), see comment in Interp.Main
 - Optimize `Interp.Options` calls, now called per display. Cache per eval? needs to handle nested evals.
 - `<array decode value>[{start: ...: end: ...}]` syntax a bit broken.
+- REPL completion might have side effcts. Make interp.Function type know and wrap somehow? input, inputs, open, ...
 
 ### TODO and ideas
 

doc/usage.md

@@ -418,6 +418,8 @@ you currently have to do `fq -d raw 'mp3({force: true})' file`.
 - `p`/`preview` show preview of field tree
 - `hd`/`hexdump` hexdump value
 - `repl` nested REPL, must be last in a pipeline. `1 | repl`, can "slurp" outputs `1, 2, 3 | repl`.
+- `paste` read string from stdin until ^D. Useful for pasting text.
+    - Ex: `paste | frompem | asn1_ber | repl` read from stdin then decode and start a new sub-REPL with result.
 
 ## Color and unicode output
 

internal/script/script.go

@@ -149,8 +149,8 @@ func (cr *CaseRun) ConfigDir() (string, error) { return "/config", nil }
 
 func (cr *CaseRun) FS() fs.FS { return cr.Case }
 
-func (cr *CaseRun) Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) {
-	cr.ActualStdoutBuf.WriteString(prompt)
+func (cr *CaseRun) Readline(opts interp.ReadlineOpts) (string, error) {
+	cr.ActualStdoutBuf.WriteString(opts.Prompt)
 	if cr.ReadlinesPos >= len(cr.Readlines) {
 		return "", io.EOF
 	}
@@ -165,7 +165,7 @@ func (cr *CaseRun) Readline(prompt string, complete func(line string, pos int) (
 		cr.ActualStdoutBuf.WriteString(lineRaw + "\n")
 
 		l := len(line) - 1
-		newLine, shared := complete(line[0:l], l)
+		newLine, shared := opts.CompleteFn(line[0:l], l)
 		// TODO: shared
 		_ = shared
 		for _, nl := range newLine {

pkg/cli/cli.go

@@ -158,7 +158,7 @@ func (stdOSFS) Open(name string) (fs.File, error) { return os.Open(name) }
 
 func (*stdOS) FS() fs.FS { return stdOSFS{} }
 
-func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (newLine []string, shared int)) (string, error) {
+func (o *stdOS) Readline(opts interp.ReadlineOpts) (string, error) {
 	if o.rl == nil {
 		var err error
 
@@ -179,9 +179,9 @@ func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (new
 		}
 	}
 
-	if complete != nil {
+	if opts.CompleteFn != nil {
 		o.rl.Config.AutoComplete = autoCompleterFn(func(line []rune, pos int) (newLine [][]rune, length int) {
-			names, shared := complete(string(line), pos)
+			names, shared := opts.CompleteFn(string(line), pos)
 			var runeNames [][]rune
 			for _, name := range names {
 				runeNames = append(runeNames, []rune(name[shared:]))
@@ -191,7 +191,7 @@ func (o *stdOS) Readline(prompt string, complete func(line string, pos int) (new
 		})
 	}
 
-	o.rl.SetPrompt(prompt)
+	o.rl.SetPrompt(opts.Prompt)
 	line, err := o.rl.Readline()
 	if errors.Is(err, readline.ErrInterrupt) {
 		return "", interp.ErrInterrupt

pkg/interp/binary.go

@@ -16,13 +16,14 @@ import (
 	"github.com/wader/fq/internal/progressreadseeker"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/ranges"
+	"github.com/wader/gojq"
 )
 
 func init() {
 	functionRegisterFns = append(functionRegisterFns, func(i *Interp) []Function {
 		return []Function{
 			{"_tobits", 3, 3, i._toBits, nil},
-			{"open", 0, 0, i._open, nil},
+			{"open", 0, 0, nil, i._open},
 		}
 	})
 }
@@ -175,7 +176,11 @@ func (of *openFile) ToBinary() (Binary, error) {
 // def open: #:: string| => binary
 // opens a file for reading from filesystem
 // TODO: when to close? when br loses all refs? need to use finalizer somehow?
-func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
+func (i *Interp) _open(c interface{}, a []interface{}) gojq.Iter {
+	if i.evalContext.isCompleting {
+		return gojq.NewIter()
+	}
+
 	var err error
 	var f fs.File
 	var path string
@@ -187,11 +192,11 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
 	default:
 		path, err = toString(c)
 		if err != nil {
-			return fmt.Errorf("%s: %w", path, err)
+			return gojq.NewIter(fmt.Errorf("%s: %w", path, err))
 		}
 		f, err = i.os.FS().Open(path)
 		if err != nil {
-			return err
+			return gojq.NewIter(err)
 		}
 	}
 
@@ -201,7 +206,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
 	fFI, err := f.Stat()
 	if err != nil {
 		f.Close()
-		return err
+		return gojq.NewIter(err)
 	}
 
 	// ctxreadseeker is used to make sure any io calls can be canceled
@@ -219,7 +224,7 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
 		buf, err := ioutil.ReadAll(ctxreadseeker.New(i.evalContext.ctx, &ioextra.ReadErrSeeker{Reader: f}))
 		if err != nil {
 			f.Close()
-			return err
+			return gojq.NewIter(err)
 		}
 		fRS = bytes.NewReader(buf)
 		bEnd = int64(len(buf))
@@ -246,10 +251,10 @@ func (i *Interp) _open(c interface{}, a []interface{}) interface{} {
 
 	bbf.br = bitio.NewIOBitReadSeeker(aheadRs)
 	if err != nil {
-		return err
+		return gojq.NewIter(err)
 	}
 
-	return bbf
+	return gojq.NewIter(bbf)
 }
 
 var _ Value = Binary{}

pkg/interp/decode.go

@@ -162,7 +162,7 @@ func (i *Interp) _decode(c interface{}, a []interface{}) interface{} {
 					c,
 					opts.Progress,
 					nil,
-					ioextra.DiscardCtxWriter{Ctx: i.evalContext.ctx},
+					EvalOpts{output: ioextra.DiscardCtxWriter{Ctx: i.evalContext.ctx}},
 				)
 			}
 			lastProgress := time.Now()

pkg/interp/funcs.jq

@@ -305,3 +305,14 @@ def topem($label):
   | join("\n")
   );
 def topem: topem("");
+
+def paste:
+  if _is_completing | not then
+    ( [ _repeat_break(
+          try _stdin(64*1024)
+          catch if . == "eof" then error("break") end
+        )
+      ]
+    | join("")
+    )
+  end;