-
Notifications
You must be signed in to change notification settings - Fork 0
/
file.go
52 lines (38 loc) · 901 Bytes
/
file.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
package writers
import (
"encoding/json"
"github.com/jjmschofield/gocrawl/internal/pages"
"log"
"os"
"path"
)
type FileWriter struct {
FilePath string
pageFile *os.File
encoder *json.Encoder
}
func (w *FileWriter) Start(in chan pages.Page) {
workingDir, err := os.Getwd()
if err != nil {
log.Panicf("Can't get working directory!, %v", err)
}
filePath := path.Join(workingDir, w.FilePath)
err = os.MkdirAll (filePath, os.ModePerm)
if err != nil {
log.Panicf("Can't create directory!, %v", err)
}
pageFile, err := os.Create(path.Join(filePath, "pages.jsonl"))
if err != nil {
log.Panicf("Can't open file to write results to!, %v", err)
}
w.encoder = json.NewEncoder(pageFile)
for page := range in {
w.write(page)
}
}
func (w *FileWriter) write(page pages.Page) {
err := w.encoder.Encode(page)
if err != nil {
log.Panicf("Can't write entry! %v", err)
}
}