Permalink
Browse files

Read files in a separate goroutine, one file at once.

It prevents creation of multiple threads which are blocked waiting for file I/O.
  • Loading branch information...
1 parent ed57256 commit 01bee80d08c1eed942ee59fd2d7958821e97e699 @nsf committed Mar 4, 2012
Showing with 53 additions and 6 deletions.
  1. +3 −4 declcache.go
  2. +1 −2 package.go
  3. +49 −0 utils.go
View
@@ -5,7 +5,6 @@ import (
"go/ast"
"go/parser"
"go/token"
- "io/ioutil"
"os"
"path/filepath"
"runtime"
@@ -100,12 +99,12 @@ func (f *decl_file_cache) update() {
}
f.mtime = statmtime
- f.read_file(f.name)
+ f.read_file()
}
-func (f *decl_file_cache) read_file(filename string) {
+func (f *decl_file_cache) read_file() {
var data []byte
- data, f.error = ioutil.ReadFile(f.name)
+ data, f.error = file_reader.read_file(f.name)
if f.error != nil {
return
}
View
@@ -7,7 +7,6 @@ import (
"go/ast"
"go/token"
"io"
- "io/ioutil"
"os"
"strconv"
"text/scanner"
@@ -84,7 +83,7 @@ func (m *package_file_cache) update_cache() {
if m.mtime != statmtime {
m.mtime = statmtime
- data, err := ioutil.ReadFile(fname)
+ data, err := file_reader.read_file(fname)
if err != nil {
return
}
View
@@ -3,6 +3,7 @@ package main
import (
"bytes"
"fmt"
+ "io/ioutil"
"os"
"path/filepath"
"runtime"
@@ -70,3 +71,51 @@ func print_backtrace(err interface{}) {
}
fmt.Println("")
}
+
+//-------------------------------------------------------------------------
+// File reader goroutine
+//
+// It's a bad idea to block multiple goroutines on file I/O. Creates many
+// threads which fight for HDD. Therefore only single goroutine should read HDD
+// at the same time.
+// -------------------------------------------------------------------------
+
+type file_read_request struct {
+ filename string
+ out chan file_read_response
+}
+
+type file_read_response struct {
+ data []byte
+ error error
+}
+
+type file_reader_type struct {
+ in chan file_read_request
+}
+
+func new_file_reader() *file_reader_type {
+ this := new(file_reader_type)
+ this.in = make(chan file_read_request)
+ go func() {
+ var rsp file_read_response
+ for {
+ req := <-this.in
+ rsp.data, rsp.error = ioutil.ReadFile(req.filename)
+ req.out <- rsp
+ }
+ }()
+ return this
+}
+
+func (this *file_reader_type) read_file(filename string) ([]byte, error) {
+ req := file_read_request{
+ filename,
+ make(chan file_read_response),
+ }
+ this.in <- req
+ rsp := <-req.out
+ return rsp.data, rsp.error
+}
+
+var file_reader = new_file_reader()

0 comments on commit 01bee80

Please sign in to comment.