forked from xitongsys/parquet-go
-
Notifications
You must be signed in to change notification settings - Fork 0
/
json.go
46 lines (40 loc) · 1.29 KB
/
json.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
package writer
import (
"io"
"github.com/xitongsys/parquet-go-source/writerfile"
"github.com/xitongsys/parquet-go/layout"
"github.com/xitongsys/parquet-go/marshal"
"github.com/xitongsys/parquet-go/parquet"
"github.com/xitongsys/parquet-go/schema"
"github.com/xitongsys/parquet-go/source"
)
type JSONWriter struct {
ParquetWriter
}
func NewJSONWriterFromWriter(jsonSchema string, w io.Writer, np int64) (*JSONWriter, error) {
wf := writerfile.NewWriterFile(w)
return NewJSONWriter(jsonSchema, wf, np)
}
//Create JSON writer
func NewJSONWriter(jsonSchema string, pfile source.ParquetFile, np int64) (*JSONWriter, error) {
var err error
res := new(JSONWriter)
res.SchemaHandler, err = schema.NewSchemaHandlerFromJSON(jsonSchema)
if err != nil {
return res, err
}
res.PFile = pfile
res.PageSize = 8 * 1024 //8K
res.RowGroupSize = 128 * 1024 * 1024 //128M
res.CompressionType = parquet.CompressionCodec_SNAPPY
res.PagesMapBuf = make(map[string][]*layout.Page)
res.DictRecs = make(map[string]*layout.DictRecType)
res.NP = np
res.Footer = parquet.NewFileMetaData()
res.Footer.Version = 1
res.Footer.Schema = append(res.Footer.Schema, res.SchemaHandler.SchemaElements...)
res.Offset = 4
_, err = res.PFile.Write([]byte("PAR1"))
res.MarshalFunc = marshal.MarshalJSON
return res, err
}