forked from xitongsys/parquet-go-source
-
Notifications
You must be signed in to change notification settings - Fork 0
/
hdfs.go
107 lines (96 loc) · 2.04 KB
/
hdfs.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
package hdfs
import (
"github.com/colinmarc/hdfs/v2"
"github.com/xitongsys/parquet-go/source"
)
type HdfsFile struct {
Hosts []string
User string
Client *hdfs.Client
FilePath string
FileReader *hdfs.FileReader
FileWriter *hdfs.FileWriter
}
func NewHdfsFileWriter(hosts []string, user string, name string) (source.ParquetFile, error) {
res := &HdfsFile{
Hosts: hosts,
User: user,
FilePath: name,
}
return res.Create(name)
}
func NewHdfsFileReader(hosts []string, user string, name string) (source.ParquetFile, error) {
res := &HdfsFile{
Hosts: hosts,
User: user,
FilePath: name,
}
return res.Open(name)
}
func (self *HdfsFile) Create(name string) (source.ParquetFile, error) {
var err error
hf := new(HdfsFile)
hf.Hosts = self.Hosts
hf.User = self.User
hf.Client, err = hdfs.NewClient(hdfs.ClientOptions{
Addresses: hf.Hosts,
User: hf.User,
})
hf.FilePath = name
if err != nil {
return hf, err
}
hf.FileWriter, err = hf.Client.Create(name)
return hf, err
}
func (self *HdfsFile) Open(name string) (source.ParquetFile, error) {
var (
err error
)
if name == "" {
name = self.FilePath
}
hf := new(HdfsFile)
hf.Hosts = self.Hosts
hf.User = self.User
hf.Client, err = hdfs.NewClient(hdfs.ClientOptions{
Addresses: hf.Hosts,
User: hf.User,
})
hf.FilePath = name
if err != nil {
return hf, err
}
hf.FileReader, err = hf.Client.Open(name)
return hf, err
}
func (self *HdfsFile) Seek(offset int64, pos int) (int64, error) {
return self.FileReader.Seek(offset, pos)
}
func (self *HdfsFile) Read(b []byte) (cnt int, err error) {
var n int
ln := len(b)
for cnt < ln {
n, err = self.FileReader.Read(b[cnt:])
cnt += n
if err != nil {
break
}
}
return cnt, err
}
func (self *HdfsFile) Write(b []byte) (n int, err error) {
return self.FileWriter.Write(b)
}
func (self *HdfsFile) Close() error {
if self.FileReader != nil {
self.FileReader.Close()
}
if self.FileWriter != nil {
self.FileWriter.Close()
}
if self.Client != nil {
self.Client.Close()
}
return nil
}