#167 Lines longer than 64k cause error on scan

script.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"math"
 	"net/http"
 	"os"
 	"os/exec"
@@ -35,6 +36,11 @@ type Pipe struct {
 	err error
 }
 
+const (
+	scannerMaxScanTokenSize = math.MaxInt // Maximum size used to buffer a bufio.Scanner token.
+	scannerStartBufSize     = 4096        // Size of initial allocation for bufio.Scanner buffer.
+)
+
 // Args creates a pipe containing the program's command-line arguments from
 // [os.Args], excluding the program name, one per line.
 func Args() *Pipe {
@@ -339,6 +345,7 @@ func (p *Pipe) Do(req *http.Request) *Pipe {
 func (p *Pipe) EachLine(process func(string, *strings.Builder)) *Pipe {
 	return p.Filter(func(r io.Reader, w io.Writer) error {
 		scanner := bufio.NewScanner(r)
+		scanner.Buffer(make([]byte, scannerStartBufSize), scannerMaxScanTokenSize)
 		output := strings.Builder{}
 		for scanner.Scan() {
 			process(scanner.Text(), &output)
@@ -414,6 +421,7 @@ func (p *Pipe) ExecForEach(cmdLine string) *Pipe {
 	}
 	return p.Filter(func(r io.Reader, w io.Writer) error {
 		scanner := bufio.NewScanner(r)
+		scanner.Buffer(make([]byte, scannerStartBufSize), scannerMaxScanTokenSize)
 		for scanner.Scan() {
 			cmdLine := strings.Builder{}
 			err := tpl.Execute(&cmdLine, scanner.Text())
@@ -503,6 +511,7 @@ func (p *Pipe) FilterLine(filter func(string) string) *Pipe {
 func (p *Pipe) FilterScan(filter func(string, io.Writer)) *Pipe {
 	return p.Filter(func(r io.Reader, w io.Writer) error {
 		scanner := bufio.NewScanner(r)
+		scanner.Buffer(make([]byte, scannerStartBufSize), scannerMaxScanTokenSize)
 		for scanner.Scan() {
 			filter(scanner.Text(), w)
 		}
@@ -556,6 +565,7 @@ func (p *Pipe) Freq() *Pipe {
 	}
 	return p.Filter(func(r io.Reader, w io.Writer) error {
 		scanner := bufio.NewScanner(r)
+		scanner.Buffer(make([]byte, scannerStartBufSize), scannerMaxScanTokenSize)
 		for scanner.Scan() {
 			freq[scanner.Text()]++
 		}
@@ -598,6 +608,7 @@ func (p *Pipe) Get(URL string) *Pipe {
 func (p *Pipe) Join() *Pipe {
 	return p.Filter(func(r io.Reader, w io.Writer) error {
 		scanner := bufio.NewScanner(r)
+		scanner.Buffer(make([]byte, scannerStartBufSize), scannerMaxScanTokenSize)
 		var line string
 		first := true
 		for scanner.Scan() {
@@ -660,6 +671,7 @@ func (p *Pipe) Last(n int) *Pipe {
 	}
 	return p.Filter(func(r io.Reader, w io.Writer) error {
 		scanner := bufio.NewScanner(r)
+		scanner.Buffer(make([]byte, scannerStartBufSize), scannerMaxScanTokenSize)
 		input := ring.New(n)
 		for scanner.Scan() {
 			input.Value = scanner.Text()

script_test.go

@@ -467,6 +467,22 @@ func TestFilterScan_FiltersInputLineByLine(t *testing.T) {
 	}
 }
 
+func TestFilterScanHandlesLongLines(t *testing.T) {
+	t.Parallel()
+	want := "short line\n"
+	got, err := script.File("testdata/longlines.txt").FilterScan(func(line string, w io.Writer) {
+		if strings.HasPrefix(line, "s") {
+			fmt.Fprintln(w, line)
+		}
+	}).String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want != got {
+		t.Error(cmp.Diff(want, got))
+	}
+}
+
 func TestFirstDropsAllButFirstNLinesOfInput(t *testing.T) {
 	t.Parallel()
 	input := "a\nb\nc\n"
@@ -764,6 +780,18 @@ func TestLastHasNoEffectGivenLessThanNInputLines(t *testing.T) {
 	}
 }
 
+func TestLastHandlesLongLines(t *testing.T) {
+	t.Parallel()
+	want := "last line\n"
+	got, err := script.File("testdata/longlines.txt").Last(1).String()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want != got {
+		t.Error(cmp.Diff(want, got))
+	}
+}
+
 func TestMatchOutputsOnlyMatchingLinesOfInput(t *testing.T) {
 	t.Parallel()
 	input := "This is the first line in the file.\nHello, world.\nThis is another line in the file.\n"