-
Notifications
You must be signed in to change notification settings - Fork 342
/
stream.go
156 lines (137 loc) · 3.69 KB
/
stream.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
package s3
import (
"bytes"
"encoding/hex"
"errors"
"fmt"
"io"
"time"
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
"github.com/treeverse/lakefs/pkg/logging"
)
const (
StreamingSha256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
StreamingContentEncoding = "aws-chunked"
)
type StreamingReader struct {
Reader io.Reader
Size int64
StreamSigner *v4.StreamSigner
Time time.Time
ChunkSize int
ChunkTimeout time.Duration
currentChunk io.Reader
totalRead int64
eof bool
}
func chunkBoundary(signature []byte, length int) []byte {
return []byte(fmt.Sprintf("%x;chunk-signature=%s\r\n", length, hex.EncodeToString(signature)))
}
func isEOF(err error) bool {
return err == io.EOF || errors.Is(err, io.ErrUnexpectedEOF)
}
func (s *StreamingReader) GetLastChunk() []byte {
res := make([]byte, 0)
sig, _ := s.StreamSigner.GetSignature([]byte{}, []byte{}, s.Time)
lastBoundary := chunkBoundary(sig, 0)
res = append(res, lastBoundary...)
res = append(res, '\r', '\n') // additional \r\n after the last boundary
return res
}
// ReadAllWithTimeout is taken from io.ReadAtLeast and adapted to support a timeout and a minimum read size
func ReadAllWithTimeout(r io.Reader, buf []byte, timeout time.Duration, minSize int) (n int, err error) {
desired := len(buf)
lg := logging.ContextUnavailable().WithFields(logging.Fields{
"timeout": timeout,
"desired_size": desired,
})
start := time.Now()
timedOut := false
for n < desired && err == nil {
var nn int
nn, err = r.Read(buf[n:])
n += nn
if time.Since(start) > timeout && n >= minChunkSize {
timedOut = true
break
}
}
if n >= desired {
err = nil
} else if n > 0 && err == io.EOF {
err = io.ErrUnexpectedEOF
}
if err == nil && timedOut {
err = ErrReaderTimeout
lg.WithField("n", n).Warn("duration passed, reader timed out")
}
return
}
var ErrReaderTimeout = errors.New("reader timeout")
func (s *StreamingReader) readNextChunk() error {
buf := make([]byte, s.ChunkSize)
n, err := ReadAllWithTimeout(s.Reader, buf, s.ChunkTimeout, minChunkSize)
s.totalRead += int64(n)
buf = buf[:n]
if err != nil && !isEOF(err) && !errors.Is(err, ErrReaderTimeout) {
// actual error happened
return err
}
if n == 0 {
if s.currentChunk == nil {
s.currentChunk = bytes.NewBuffer(s.GetLastChunk())
}
return io.EOF
}
sig, sigErr := s.StreamSigner.GetSignature([]byte{}, buf, s.Time)
if sigErr != nil {
return sigErr
}
buf = append(buf, '\r', '\n') // additional \r\n after the content
boundary := chunkBoundary(sig, n)
if isEOF(err) || s.totalRead == s.Size {
// we're done with the upstream Reader, let's write one last chunk boundary.
buf = append(buf, s.GetLastChunk()...)
}
buf = append(boundary, buf...)
s.currentChunk = bytes.NewBuffer(buf)
if s.totalRead == s.Size {
err = io.EOF
}
if errors.Is(err, ErrReaderTimeout) {
return nil // a slow reader shouldn't fail.
}
return err
}
func (s *StreamingReader) Read(p []byte) (int, error) {
if s.eof {
return 0, io.EOF
}
var nerr error
var currentN int
if s.currentChunk == nil {
nerr = s.readNextChunk()
if nerr != nil && !isEOF(nerr) {
return currentN, nerr
}
}
for currentN < len(p) {
n, err := s.currentChunk.Read(p[currentN:])
currentN += n
if err != nil && !isEOF(err) {
return currentN, err
}
if isEOF(err) { // we drained the current chunk
if isEOF(nerr) { // no more chunks to read
s.eof = true
return currentN, io.EOF
}
// otherwise, we read the entire chunk, let's fill it back up (if n < size of p)
nerr = s.readNextChunk()
if nerr != nil && !isEOF(nerr) {
return currentN, nerr // something bad happened when reading the next chunk
}
}
}
return currentN, nil
}