/
parquet2json.go
128 lines (108 loc) · 2.67 KB
/
parquet2json.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
/*
* Minio Cloud Storage, (C) 2018 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"encoding/json"
"fmt"
"io"
"os"
"path"
"strings"
"github.com/minio/minio-go/v7/pkg/set"
parquet "github.com/cdbarbosa/clone/pkg/s3select/internal/parquet-go"
)
func getReader(name string, offset int64, length int64) (io.ReadCloser, error) {
file, err := os.Open(name)
if err != nil {
return nil, err
}
fi, err := file.Stat()
if err != nil {
return nil, err
}
if offset < 0 {
offset = fi.Size() + offset
}
if _, err = file.Seek(offset, io.SeekStart); err != nil {
return nil, err
}
return file, nil
}
func printUsage() {
progName := path.Base(os.Args[0])
fmt.Printf("Usage: %v PARQUET-FILE [COLUMN...]\n", progName)
fmt.Println()
fmt.Printf("Examples:\n")
fmt.Printf("# Convert all columns to JSON\n")
fmt.Printf("$ %v example.parquet\n", progName)
fmt.Println()
fmt.Printf("# Convert specific columns to JSON\n")
fmt.Printf("$ %v example.par firstname dob\n", progName)
fmt.Println()
}
func main() {
if len(os.Args) < 2 {
printUsage()
os.Exit(-1)
}
name := os.Args[1]
ext := path.Ext(name)
jsonFilename := name + ".json"
if ext == ".parquet" || ext == ".par" {
jsonFilename = strings.TrimSuffix(name, ext) + ".json"
}
columns := set.CreateStringSet(os.Args[2:]...)
if len(columns) == 0 {
columns = nil
}
file, err := parquet.NewReader(
func(offset, length int64) (io.ReadCloser, error) {
return getReader(name, offset, length)
},
columns,
)
if err != nil {
fmt.Printf("%v: %v\n", name, err)
os.Exit(1)
}
defer file.Close()
jsonFile, err := os.OpenFile(jsonFilename, os.O_RDWR|os.O_CREATE, 0755)
if err != nil {
fmt.Printf("%v: %v\n", jsonFilename, err)
os.Exit(1)
}
defer jsonFile.Close()
for {
record, err := file.Read()
if err != nil {
if err != io.EOF {
fmt.Printf("%v: %v\n", name, err)
os.Exit(1)
}
break
}
data, err := json.Marshal(record)
if err != nil {
fmt.Printf("%v: %v\n", name, err)
os.Exit(1)
}
data = append(data, byte('\n'))
if _, err = jsonFile.Write(data); err != nil {
fmt.Printf("%v: %v\n", jsonFilename, err)
os.Exit(1)
}
}
}