Add a &null option to from-lines and to-lines

This change, combined with other changes to `edit:command-history`,
makes feeding its output into `fzf` extremely fast. Which makes it a
practical alternative to using the builtin Ctrl-R binding for searching
command history. It's also just generally useful to have efficient ways
to process "lines" that are null terminated rather than newline (or
cr-nl on Windows).

Resolves elves#1070
Related elves#1053

pkg/eval/builtin_fn_io.go

@@ -580,24 +580,34 @@ func slurp(fm *Frame) (string, error) {
 //elvdoc:fn from-lines
 //
 // ```elvish
-// from-lines
+// from-lines &null
 // ```
 //
-// Splits byte input into lines, and writes them to the value output. Value
-// input is ignored.
+// Splits byte input into lines, and writes them to the value output. Value input is ignored. If the
+// `&null` option is true the lines are assumed to be null terminated; the default is whatever
+// termination is normally used for lines of text on your system (e.g., newline).
 //
 // ```elvish-transcript
 // ~> { echo a; echo b } | from-lines
 // ▶ a
 // ▶ b
+// ~> print "a\000b" | from-lines
+// ▶ "a\x00b"
+// ~> print "a\000b" | from-lines &null
+// ▶ a
+// ▶ b
 // ~> { echo a; put b } | from-lines
 // ▶ a
 // ```
 //
 // @cf to-lines
 
-func fromLines(fm *Frame) {
-	linesToChan(fm.InputFile(), fm.OutputChan())
+type linesOpt struct{ Null bool }
+
+func (o *linesOpt) SetDefaultOptions() {}
+
+func fromLines(fm *Frame, opts linesOpt) {
+	linesToChan(fm.InputFile(), fm.OutputChan(), opts.Null)
 }
 
 //elvdoc:fn from-json
@@ -690,11 +700,14 @@ func fromJSONInterface(v interface{}) (interface{}, error) {
 //elvdoc:fn to-lines
 //
 // ```elvish
-// to-lines $input?
+// to-lines &null $input?
 // ```
 //
-// Writes each value input to a separate line in the byte output. Byte input is
-// ignored.
+// Writes each value input to a separate line in the byte output. Byte input is ignored. If the
+// `&null` option is true the lines are null terminated; the default is whatever line termination is
+// normally used for lines of text on your system (e.g., newline). This behavior is useful when
+// feeding the output into a program that accepts null terminated lines to avoid ambiguities if the
+// values contains newline characters.
 //
 // ```elvish-transcript
 // ~> put a b | to-lines
@@ -710,12 +723,18 @@ func fromJSONInterface(v interface{}) (interface{}, error) {
 //
 // @cf from-lines
 
-func toLines(fm *Frame, inputs Inputs) {
+func toLines(fm *Frame, opts linesOpt, inputs Inputs) {
 	out := fm.OutputFile()
 
-	inputs(func(v interface{}) {
-		fmt.Fprintln(out, vals.ToString(v))
-	})
+	if opts.Null {
+		inputs(func(v interface{}) {
+			fmt.Fprint(out, vals.ToString(v), "\000")
+		})
+	} else {
+		inputs(func(v interface{}) {
+			fmt.Fprintln(out, vals.ToString(v))
+		})
+	}
 }
 
 //elvdoc:fn to-json

pkg/eval/builtin_fn_io_test.go

@@ -84,6 +84,7 @@ func TestFromLines(t *testing.T) {
 	Test(t,
 		That(`print "a\nb" | from-lines`).Puts("a", "b"),
 		That(`print "a\nb\n" | from-lines`).Puts("a", "b"),
+		That(`print "a\nb\x00\x00c\x00d" | from-lines &null`).Puts("a\nb", "", "c", "d"),
 	)
 }
 
@@ -101,6 +102,7 @@ func TestFromJson(t *testing.T) {
 func TestToLines(t *testing.T) {
 	Test(t,
 		That(`put "l\norem" ipsum | to-lines`).Prints("l\norem\nipsum\n"),
+		That(`put "l\norem" ipsum | to-lines &null`).Prints("l\norem\x00ipsum\x00"),
 	)
 }
 

pkg/eval/frame.go

@@ -117,7 +117,7 @@ func (fm *Frame) IterateInputs(f func(interface{})) {
 
 	w.Add(2)
 	go func() {
-		linesToChan(fm.InputFile(), inputs)
+		linesToChan(fm.InputFile(), inputs, false)
 		w.Done()
 	}()
 	go func() {
@@ -136,12 +136,18 @@ func (fm *Frame) IterateInputs(f func(interface{})) {
 	}
 }
 
-func linesToChan(r io.Reader, ch chan<- interface{}) {
+func linesToChan(r io.Reader, ch chan<- interface{}, nullTerminated bool) {
+	lineTerminator := byte('\n')
+	chopLineEnding := strutil.ChopLineEnding
+	if nullTerminated {
+		lineTerminator = 0
+		chopLineEnding = strutil.ChopNullEnding
+	}
 	filein := bufio.NewReader(r)
 	for {
-		line, err := filein.ReadString('\n')
+		line, err := filein.ReadString(lineTerminator)
 		if line != "" {
-			ch <- strutil.ChopLineEnding(line)
+			ch <- chopLineEnding(line)
 		}
 		if err != nil {
 			if err != io.EOF {

pkg/strutil/chop.go

@@ -10,3 +10,10 @@ func ChopLineEnding(s string) string {
 	}
 	return s
 }
+
+func ChopNullEnding(s string) string {
+	if len(s) >= 1 && s[len(s)-1] == '\x00' {
+		return s[:len(s)-1]
+	}
+	return s
+}