This repository has been archived by the owner on Nov 17, 2022. It is now read-only.
-
-
Notifications
You must be signed in to change notification settings - Fork 3
/
ndjson.go
96 lines (87 loc) · 1.95 KB
/
ndjson.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
package io
import (
"bufio"
"errors"
gogoio "github.com/gogo/protobuf/io"
"github.com/gogo/protobuf/jsonpb"
"github.com/gogo/protobuf/proto"
"io"
"strings"
)
func NewNdjsonWriter(writer io.Writer) gogoio.WriteCloser {
marshaler := jsonpb.Marshaler{
EnumsAsInts: false,
EmitDefaults: false,
}
return &ndjsonWriter{writer, marshaler}
}
type ndjsonWriter struct {
writer io.Writer
marshaler jsonpb.Marshaler
}
func (this *ndjsonWriter) WriteMsg(msg proto.Message) (err error) {
msgJson, err := this.marshaler.MarshalToString(msg)
if err != nil {
return err
}
out := bufio.NewWriter(this.writer)
_, err = out.WriteString(msgJson)
if err != nil {
return err
}
_, err = out.WriteString("\n")
if err != nil {
return err
}
err = out.Flush()
if err != nil {
return err
}
return nil
}
func (this *ndjsonWriter) Close() error {
if closer, ok := this.writer.(io.Closer); ok {
return closer.Close()
}
return nil
}
func NewNdjsonReader(reader io.Reader) gogoio.ReadCloser {
var closer io.Closer
if c, ok := reader.(io.Closer); ok {
closer = c
}
scanner := bufio.NewScanner(reader)
const maxCapacity = 10 * 1024 * 1024 // 10Mb
buf := make([]byte, maxCapacity)
scanner.Buffer(buf, maxCapacity)
unmarshaler := jsonpb.Unmarshaler{
AllowUnknownFields: true,
}
return &ndjsonReader{*bufio.NewReader(reader), *scanner, unmarshaler, closer}
}
type ndjsonReader struct {
reader bufio.Reader
scanner bufio.Scanner
unmarshaler jsonpb.Unmarshaler
closer io.Closer
}
func (this *ndjsonReader) ReadMsg(msg proto.Message) error {
if this.scanner.Scan() {
line := strings.TrimSpace(this.scanner.Text())
if len(line) == 0 {
return this.ReadMsg(msg)
}
err := this.unmarshaler.Unmarshal(strings.NewReader(line), msg)
if err != nil {
return errors.New("Not JSON: " + line)
}
return nil
}
return io.EOF
}
func (this *ndjsonReader) Close() error {
if this.closer != nil {
return this.closer.Close()
}
return nil
}