-
Notifications
You must be signed in to change notification settings - Fork 0
/
encoder.go
259 lines (224 loc) · 7.42 KB
/
encoder.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
package stitch
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"fmt"
"io"
"os"
aesgcm "github.com/OhanaFS/stitch/aes"
"github.com/OhanaFS/stitch/header"
"github.com/OhanaFS/stitch/reedsolomon"
seekable "github.com/SaveTheRbtz/zstd-seekable-format-go"
"github.com/hashicorp/vault/shamir"
"github.com/klauspost/compress/zstd"
)
// splitFileKey encrypts the file key with the given key and iv, and splits it
// into the given number of shards.
func splitFileKey(fileKey, key, iv []byte, shards, threshold int) ([][]byte, error) {
// Encrypt the file key with the given key and iv.
block, err := aes.NewCipher(key)
if err != nil {
return nil, fmt.Errorf("failed to create AES cipher: %v", err)
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return nil, fmt.Errorf("failed to create AES-GCM: %v", err)
}
fileKeyCiphertext := gcm.Seal(nil, iv, fileKey, nil)
// Split the key into shards.
fileKeySplit, err := shamir.Split(
fileKeyCiphertext, shards, threshold,
)
if err != nil {
return nil, fmt.Errorf("failed to generate parts: %v", err)
}
return fileKeySplit, nil
}
// Encode takes in a reader, performs the transformations and then splits the
// data into multiple shards, writing them to the output writers. The output
// writers are not closed after the data is written.
//
// After the data has finished encoding, a header will be written to the end of
// each shard. At this point, the shards are not usable yet until the header is
// finalized using the FinalizeHeader() function.
func (e *Encoder) Encode(data io.Reader, shards []io.Writer, key []byte, iv []byte) (*EncodingResult, error) {
totalShards := int(e.opts.DataShards + e.opts.ParityShards)
// Check if the number of output writers matches the number of shards in the
// encoder options.
if len(shards) != totalShards {
return nil, ErrShardCountMismatch
}
// Prepare a 256-bit AES key to encrypt the data.
fileKey := make([]byte, 32)
if _, err := rand.Read(fileKey); err != nil {
return nil, fmt.Errorf("failed to generate file key: %v", err)
}
// Encrypt and split the key into the number of shards.
fileKeySplit, err := splitFileKey(fileKey, key, iv, totalShards, int(e.opts.KeyThreshold))
if err != nil {
return nil, fmt.Errorf("failed to split file key: %v", err)
}
// Prepare headers for each shard.
headers := make([]header.Header, totalShards)
for i := 0; i < totalShards; i++ {
headers[i] = header.Header{
ShardIndex: i,
ShardCount: totalShards,
FileKey: fileKeySplit[i],
FileHash: make([]byte, 32),
FileSize: 0,
EncryptedSize: 0,
CompressedSize: 0,
RSBlockSize: rsBlockSize,
AESBlockSize: aesBlockSize,
IsComplete: false,
}
// Write the header to the shard.
b, err := headers[i].Encode()
if err != nil {
return nil, fmt.Errorf("failed to encode header: %v", err)
}
if _, err := shards[i].Write(b); err != nil {
return nil, fmt.Errorf("failed to write header: %v", err)
}
}
// Prepare the Reed-Solomon encoder.
encRS, err := reedsolomon.NewEncoder(
int(e.opts.DataShards), int(e.opts.ParityShards), rsBlockSize,
)
if err != nil {
return nil, fmt.Errorf("failed to create Reed-Solomon encoder: %v", err)
}
// Prepare the Reed-Solomon writer.
wRS := reedsolomon.NewWriter(shards, encRS)
// Prepare the AES writer.
wAES, err := aesgcm.NewWriter(wRS, fileKey, aesBlockSize)
if err != nil {
return nil, fmt.Errorf("failed to create AES writer: %v", err)
}
// Prepare the zstd compressor.
encZstd, err := zstd.NewWriter(nil)
if err != nil {
return nil, fmt.Errorf("failed to create zstd writer: %v", err)
}
wZstd, err := seekable.NewWriter(wAES, encZstd)
if err != nil {
return nil, fmt.Errorf("failed to create zstd writer: %v", err)
}
// Start encoding
chunk := make([]byte, rsBlockSize)
hash := sha256.New()
fileSize := uint64(0)
for {
// Read a block of data
n, err := data.Read(chunk)
if err != nil {
if err == io.EOF {
break
}
return nil, fmt.Errorf("failed to read data: %v", err)
}
fileSize += uint64(n)
// Encode
if _, err := wZstd.Write(chunk[:n]); err != nil {
return nil, fmt.Errorf("failed to write to compressor: %v", err)
}
// Update the hash
if _, err := hash.Write(chunk[:n]); err != nil {
return nil, fmt.Errorf("failed to hash chunk: %v", err)
}
if n < rsBlockSize {
break
}
}
// Close the writers
if err := wZstd.Close(); err != nil {
return nil, fmt.Errorf("failed to close compressor writer: %v", err)
}
if err := encZstd.Close(); err != nil {
return nil, fmt.Errorf("failed to close compressor encoder: %v", err)
}
if err := wAES.Close(); err != nil {
return nil, fmt.Errorf("failed to close aes writer: %v", err)
}
if err := wRS.Close(); err != nil {
return nil, fmt.Errorf("failed to close reed-solomon writer: %v", err)
}
// Write the complete header to the end of the file.
digest := hash.Sum(nil)
for i := 0; i < totalShards; i++ {
headers[i].FileHash = digest
headers[i].FileSize = fileSize
headers[i].EncryptedSize = wAES.(*aesgcm.AESWriter).GetWritten()
headers[i].CompressedSize = wAES.(*aesgcm.AESWriter).GetRead()
headers[i].IsComplete = true
// Write the updated header to the end of the shard.
b, err := headers[i].Encode()
if err != nil {
return nil, fmt.Errorf("failed to encode header: %v", err)
}
if _, err := shards[i].Write(b); err != nil {
return nil, fmt.Errorf("failed to write header: %v", err)
}
}
return &EncodingResult{
FileSize: fileSize,
FileHash: digest,
}, nil
}
// FinalizeHeader rewrites the shard header with the one located at the end of
// the shard. If the provided shard is an *os.File, the header at the end of the
// file will be truncated.
func (e *Encoder) FinalizeHeader(shard io.ReadWriteSeeker) error {
// Seek to the start of the shard.
if _, err := shard.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek to start of shard: %v", err)
}
// Try to read the header at the start
headerBuf := make([]byte, header.HeaderSize)
if _, err := shard.Read(headerBuf); err != nil {
return fmt.Errorf("failed to read header at start of shard: %v", err)
}
// Parse the header at the start
hdr := header.NewHeader()
if err := hdr.Decode(headerBuf); err != nil {
return fmt.Errorf("failed to decode header at start of shard: %v", err)
}
// Skip if the header is already complete
if hdr.IsComplete {
return nil
}
// Seek to the end of the shard
hdrOffset, err := shard.Seek(-int64(header.HeaderSize), io.SeekEnd)
if err != nil {
return fmt.Errorf("failed to seek to end of shard: %v", err)
}
// Read the header at the end
if _, err := shard.Read(headerBuf); err != nil {
return fmt.Errorf("failed to read header at end of shard: %v", err)
}
// Parse the header at the end
if err := hdr.Decode(headerBuf); err != nil {
return fmt.Errorf("failed to decode header at end of shard: %v", err)
}
// Make sure the header is complete
if !hdr.IsComplete {
return header.ErrHeaderNotComplete
}
// Rewrite the header at the start
if _, err := shard.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("failed to seek to start of shard: %v", err)
}
if _, err := shard.Write(headerBuf); err != nil {
return fmt.Errorf("failed to write header at start of shard: %v", err)
}
// Try to truncate the ending header
if file, ok := shard.(*os.File); ok {
if err := file.Truncate(hdrOffset); err != nil {
return fmt.Errorf("failed to truncate shard: %v", err)
}
}
return nil
}