-
Notifications
You must be signed in to change notification settings - Fork 7
/
recordwriter.go
81 lines (68 loc) · 1.84 KB
/
recordwriter.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
package geoparquet
import (
"encoding/json"
"errors"
"fmt"
"github.com/apache/arrow/go/v14/arrow"
"github.com/apache/arrow/go/v14/parquet"
"github.com/apache/arrow/go/v14/parquet/pqarrow"
)
type RecordWriter struct {
fileWriter *pqarrow.FileWriter
metadata *Metadata
wroteGeoMetadata bool
}
func NewRecordWriter(config *WriterConfig) (*RecordWriter, error) {
parquetProps := config.ParquetWriterProps
if parquetProps == nil {
parquetProps = parquet.NewWriterProperties()
}
arrowProps := config.ArrowWriterProps
if arrowProps == nil {
defaults := pqarrow.DefaultWriterProps()
arrowProps = &defaults
}
if config.ArrowSchema == nil {
return nil, errors.New("schema is required")
}
if config.Writer == nil {
return nil, errors.New("writer is required")
}
fileWriter, fileErr := pqarrow.NewFileWriter(config.ArrowSchema, config.Writer, parquetProps, *arrowProps)
if fileErr != nil {
return nil, fileErr
}
writer := &RecordWriter{
fileWriter: fileWriter,
metadata: config.Metadata,
}
return writer, nil
}
func (w *RecordWriter) AppendKeyValueMetadata(key string, value string) error {
if err := w.fileWriter.AppendKeyValueMetadata(key, value); err != nil {
return err
}
if key == MetadataKey {
w.wroteGeoMetadata = true
}
return nil
}
func (w *RecordWriter) Write(record arrow.Record) error {
return w.fileWriter.WriteBuffered(record)
}
func (w *RecordWriter) Close() error {
if !w.wroteGeoMetadata {
metadata := w.metadata
if metadata == nil {
metadata = DefaultMetadata()
}
data, err := json.Marshal(metadata)
if err != nil {
return fmt.Errorf("failed to encode %s file metadata", MetadataKey)
}
if err := w.fileWriter.AppendKeyValueMetadata(MetadataKey, string(data)); err != nil {
return fmt.Errorf("failed to append %s file metadata", MetadataKey)
}
}
return w.fileWriter.Close()
}