/
incremental_page_reader.go
168 lines (147 loc) · 4.72 KB
/
incremental_page_reader.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
package internal
import (
"bytes"
"encoding/binary"
"io"
"github.com/RoaringBitmap/roaring"
"github.com/tinsane/tracelog"
"github.com/wal-g/wal-g/internal/ioextensions"
"github.com/wal-g/wal-g/utility"
)
// "wi" at the head stands for "wal-g increment"
// format version "1", signature magic number
var IncrementFileHeader = []byte{'w', 'i', '1', SignatureMagicNumber}
// IncrementalPageReader constructs difference map during initialization and than re-read file
// Diff map may consist of 1Gb/PostgresBlockSize elements == 512Kb
type IncrementalPageReader struct {
PagedFile ioextensions.ReadSeekCloser
FileSize int64
Lsn uint64
Next []byte
Blocks []uint32
}
func (pageReader *IncrementalPageReader) Read(p []byte) (n int, err error) {
for {
copied := copy(p, pageReader.Next)
p = p[copied:]
pageReader.Next = pageReader.Next[copied:]
n += copied
if len(p) == 0 {
return n, nil
}
moreData, err := pageReader.DrainMoreData()
if err != nil {
return n, err
}
if !moreData {
return n, io.EOF
}
}
}
func (pageReader *IncrementalPageReader) DrainMoreData() (succeed bool, err error) {
if len(pageReader.Blocks) == 0 {
return false, nil
}
err = pageReader.AdvanceFileReader()
if err != nil {
return false, err
}
return true, nil
}
func (pageReader *IncrementalPageReader) AdvanceFileReader() error {
pageBytes := make([]byte, DatabasePageSize)
blockNo := pageReader.Blocks[0]
pageReader.Blocks = pageReader.Blocks[1:]
offset := int64(blockNo) * int64(DatabasePageSize)
// TODO : possible race condition - page was deleted between blocks extraction and seek
_, err := pageReader.PagedFile.Seek(offset, io.SeekStart)
if err != nil {
return err
}
_, err = io.ReadFull(pageReader.PagedFile, pageBytes)
if err == nil {
pageReader.Next = pageBytes
}
return err
}
// Close IncrementalPageReader
func (pageReader *IncrementalPageReader) Close() error {
return pageReader.PagedFile.Close()
}
// TODO : unit tests
// TODO : "initialize" is rather meaningless name, maybe this func should be decomposed
func (pageReader *IncrementalPageReader) initialize(deltaBitmap *roaring.Bitmap) (size int64, err error) {
var headerBuffer bytes.Buffer
headerBuffer.Write(IncrementFileHeader)
fileSize := pageReader.FileSize
headerBuffer.Write(utility.ToBytes(uint64(fileSize)))
pageReader.Blocks = make([]uint32, 0, fileSize/int64(DatabasePageSize))
if deltaBitmap == nil {
err := pageReader.FullScanInitialize()
if err != nil {
return 0, err
}
} else {
pageReader.DeltaBitmapInitialize(deltaBitmap)
}
pageReader.WriteDiffMapToHeader(&headerBuffer)
pageReader.Next = headerBuffer.Bytes()
pageDataSize := int64(len(pageReader.Blocks)) * int64(DatabasePageSize)
size = int64(headerBuffer.Len()) + pageDataSize
return
}
func (pageReader *IncrementalPageReader) DeltaBitmapInitialize(deltaBitmap *roaring.Bitmap) {
it := deltaBitmap.Iterator()
for it.HasNext() { // TODO : do something with file truncation during reading
blockNo := it.Next()
if pageReader.FileSize >= int64(blockNo+1)*int64(DatabasePageSize) { // whole block fits into file
pageReader.Blocks = append(pageReader.Blocks, blockNo)
} else {
break
}
}
}
func (pageReader *IncrementalPageReader) FullScanInitialize() error {
pageBytes := make([]byte, DatabasePageSize)
for currentBlockNumber := uint32(0); ; currentBlockNumber++ {
_, err := io.ReadFull(pageReader.PagedFile, pageBytes)
if err != nil {
if err == io.EOF || err == io.ErrUnexpectedEOF {
return nil
}
return err
}
valid := pageReader.SelectNewValidPage(pageBytes, currentBlockNumber) // TODO : torn page possibility
if !valid {
return NewInvalidBlockError(currentBlockNumber)
}
}
}
// WriteDiffMapToHeader is currently used only with buffers, so we don't handle any writing errors
func (pageReader *IncrementalPageReader) WriteDiffMapToHeader(headerWriter io.Writer) {
diffBlockCount := len(pageReader.Blocks)
_, _ = headerWriter.Write(utility.ToBytes(uint32(diffBlockCount)))
for _, blockNo := range pageReader.Blocks {
_ = binary.Write(headerWriter, binary.LittleEndian, blockNo)
}
return
}
// SelectNewValidPage checks whether page is valid and if it so, then blockNo is appended to Blocks list
func (pageReader *IncrementalPageReader) SelectNewValidPage(pageBytes []byte, blockNo uint32) (valid bool) {
pageHeader, _ := ParsePostgresPageHeader(bytes.NewReader(pageBytes))
valid = pageHeader.IsValid()
isNew := false
if !valid {
if pageHeader.IsNew() { // vacuumed page
isNew = true
valid = true
} else {
tracelog.DebugLogger.Println("Invalid page ", blockNo, " page header ", pageHeader)
return false
}
}
if isNew || (pageHeader.Lsn() >= pageReader.Lsn) {
pageReader.Blocks = append(pageReader.Blocks, blockNo)
}
return
}