by striginsv@yandex.ru:
There is a problem using []byte channel when processing large amount of data.
$ wc -l bigfile.txt
3601800 bigfile.txt
package main
import (
"bufio"
"fmt"
"os"
"runtime"
)
var ch chan []byte
func reader(fname string, ch chan []byte) {
f, err := os.Open(fname)
if err != nil {
return
}
defer f.Close()
r := bufio.NewReader(f)
for {
s, ok := r.ReadSlice('\n')
if ok != nil {
close(ch)
break
}
ch <- (s)
}
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
ch := make(chan []byte)
if len(os.Args) != 2 {
fmt.Printf("Usage: %s <filename>\n", os.Args[0])
return
}
fname := os.Args[1]
go reader(fname, ch)
// num := 0
for s := range ch {
_ = s
// num++
fmt.Print(string(s))
}
// fmt.Println("Lines number :", num)
}
$ go run counter.go | wc -l
3601787
But if we uncomment counter, the output will be right:
Lines number : 3601800
The problem disappears if we change channel type to string, or disable GOMAXPROCS.
