-
Notifications
You must be signed in to change notification settings - Fork 23
/
line.go
92 lines (79 loc) · 1.63 KB
/
line.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
package index
import (
"encoding/binary"
"fmt"
"github.com/MG-RAST/Shock/shock-server/conf"
"github.com/MG-RAST/Shock/shock-server/node/file/format/line"
"io"
"math/rand"
"os"
)
type lineRecord struct {
f *os.File
r line.LineReader
t string
snf string
snp string
Index *Idx
}
func NewLineIndexer(f *os.File, nType string, snFormat string, snIndexPath string) Indexer {
return &lineRecord{
f: f,
r: line.NewReader(f),
t: nType,
snf: snFormat,
snp: snIndexPath,
Index: New(),
}
}
func (l *lineRecord) Create(file string) (count int64, format string, err error) {
tmpFilePath := fmt.Sprintf("%s/temp/%d%d.idx", conf.PATH_DATA, rand.Int(), rand.Int())
f, err := os.Create(tmpFilePath)
if err != nil {
return
}
defer f.Close()
format = "array"
eof := false
curr := int64(0)
count = 0
buffer_pos := 0 // used to track the location in our byte array
// Writing index file in 16MB chunks
var b [16777216]byte
for {
// io.EOF error does not get returned from GetReadOffset() until all lines have been read
n, er := l.r.GetReadOffset()
if er != nil {
if er != io.EOF {
err = er
return
}
eof = true
}
x := (buffer_pos * 16)
if x == 16777216 {
f.Write(b[:])
buffer_pos = 0
x = 0
}
y := x + 8
z := x + 16
binary.LittleEndian.PutUint64(b[x:y], uint64(curr))
binary.LittleEndian.PutUint64(b[y:z], uint64(n))
curr += int64(n)
count += 1
buffer_pos += 1
if eof {
break
}
}
if buffer_pos != 0 {
f.Write(b[:buffer_pos*16])
}
err = os.Rename(tmpFilePath, file)
return
}
func (l *lineRecord) Close() (err error) {
l.f.Close()
return
}