/
filecontainer.go
181 lines (158 loc) · 5.43 KB
/
filecontainer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
package shards
import (
"bytes"
"encoding/binary"
"fmt"
. "github.com/archoncloud/archoncloud-go/common"
"github.com/archoncloud/archoncloud-go/interfaces"
"io"
)
const FileContainerCurrentVersion = 0
type FileContainerType uint8
const (
NoContainer FileContainerType = iota // for non-container upload/download (raw file)
BrowserOptimized
ArchiveOptimized // using Infectious RS
)
type EncryptionType uint8
const (
NoEncryption EncryptionType = iota
)
// in bytes
const CRC32Len = 4
const RedundacyLen = 4
type FileContainer struct {
Version uint8 // currently 0
Type FileContainerType
EncryptionType EncryptionType
CompressionType uint8 // 0=no compression
Size int64 // bytes
Signature []byte
Shards *ShardsContainer
Hash []byte
}
func (fct FileContainerType) HdrLen() int {
const fileContainerCommonHdrLen = 4
switch fct {
case BrowserOptimized: return fileContainerCommonHdrLen
case ArchiveOptimized: return fileContainerCommonHdrLen+1+CRC32Len // parity+CRC32
}
return 0
}
func (fct FileContainerType) TaiLen() int {
const fileContainerCommonHdrLen = 4
switch fct {
case BrowserOptimized: return RedundacyLen+ ArchonSignatureLen
case ArchiveOptimized: return CRC32Len+ ArchonSignatureLen
}
return 0
}
func (fct FileContainerType) PayLoadEnd(fcLen int64) int64 {
tail := fct.TaiLen()
switch fct {
case BrowserOptimized:
return fcLen-int64(tail)
case ArchiveOptimized:
payloadAndParity := fcLen - int64(fct.HdrLen()+tail)
parityLen := DivideRoundUp( uint64(payloadAndParity), 26 )// 25-26 parity
return fcLen-int64(parityLen) -int64(tail)
}
return 0
}
// NewFileContainer reads the payload data from r and encodes into s
func NewFileContainer(s *ShardsContainer, r io.Reader, account interfaces.IAccount) (*FileContainer, error) {
fct := BrowserOptimized
if s.ContainerType == ArchiveOptimizedReedSolomon {
fct = ArchiveOptimized
}
f := FileContainer{
Version: FileContainerCurrentVersion,
Type: fct,
EncryptionType: NoEncryption,
CompressionType:0,
Shards: s,
}
fileContainerBytes, err := f.CreateContainerBytes(r, account)
if err != nil {return nil, err}
s.FileContainerHash = f.Hash
// This encodes the individual shards
err = s.Encode(s, fileContainerBytes.Bytes())
return &f, err
}
// CreateContainerBytes writes the file container data to w. Reads the payload data from r
func (f *FileContainer) CreateContainerBytes(r io.Reader, uploaderAccount interfaces.IAccount) (fileContainerBytes bytes.Buffer, err error) {
fileContainerBytes = bytes.Buffer{}
hashWr := NewHashingWriter(&fileContainerBytes)
hdr := []byte{byte(f.Version), byte(f.Type), byte(f.EncryptionType), byte(f.CompressionType)}
var numBytes int64 = 0;
n, err := hashWr.Write(hdr)
if err != nil {return}
numBytes += int64(n)
if f.Type == ArchiveOptimized {
// Parity for hdr
p, _ := GenerateRSIParity_4_5(hdr)
n, _ := hashWr.Write(p)
numBytes += int64(n)
n, _ = hashWr.Write(CRC32(fileContainerBytes.Bytes()))
numBytes += int64(n)
}
startOfPayload := numBytes
// Payload
nc, err := io.Copy(hashWr, r)
if err != nil {return}
numBytes += nc
if f.Type == ArchiveOptimized {
p, _ := GenerateRSIParity_25_26(fileContainerBytes.Bytes()[startOfPayload:])
_, _ = hashWr.Write(p)
crc := CRC32(fileContainerBytes.Bytes()[startOfPayload:])
_, _ = hashWr.Write(crc)
} else {
// RedundancyCheck equals the left-most 4 bytes of hash(input), where the input is an 8 byte array consisting of:
// Version, Type, EncryptionType, CompressionType and right most 4 bytes of
// the entire container size (container size % 2^32), big-endian
Size := numBytes + RedundacyLen + ArchonSignatureLen
redundancyBytes := [8]byte{ byte(f.Version), byte(f.Type), byte(f.EncryptionType), byte(f.CompressionType)}
binary.BigEndian.PutUint32(redundancyBytes[4:8], uint32(Size & 0xFFFF))
_, err = hashWr.Write(GetArchonHash(redundancyBytes[:])[:RedundacyLen])
if err != nil {return}
}
sig, err := uploaderAccount.Sign(hashWr.GetHash())
if err != nil {return}
f.Signature = sig
// Write the signature at the end
_, err = hashWr.Write(sig)
f.Hash = hashWr.GetHash()
return
}
// buf is the file container data. Returns reader of original data
func (s *ShardsContainer) GetOriginalDataFromContainer(container []byte) (io.Reader, error) {
// Note: container may include trailing zeros that are to be ignored
hdr, err := s.GetSampleShardHdr()
if err != nil {return nil,err}
containerLen := hdr.GetFileContainerLen()
if len(container) < int(containerLen) {
return nil, fmt.Errorf("file container buffer is %d bytes. expected at least %d", len(container), containerLen)
}
fct := FileContainerType(container[1])
if len(container) < fct.HdrLen() {
return nil, fmt.Errorf("file container buffer is %d bytes. expected at least %d", len(container), containerLen)
}
switch fct {
case BrowserOptimized:
case ArchiveOptimized:
// Check header CRC
crc32 := CRC32(container[:5])
if bytes.Compare(crc32, container[5:5+CRC32Len]) != 0 {
return nil, fmt.Errorf("header CRC32 does not match")
}
default:
return nil, fmt.Errorf("unknown file container type: %d", fct)
}
originHash := GetArchonHash(container[:containerLen])
if bytes.Compare(originHash, hdr.GetFileContainerHash()) != 0 {
return nil, fmt.Errorf("hash does not match original file")
}
// TODO: verification - signature, redundancy, CRC32
r := bytes.NewReader(container[fct.HdrLen() : fct.PayLoadEnd(containerLen)]);
return r, nil
}