forked from minio/minio
-
Notifications
You must be signed in to change notification settings - Fork 0
/
etag.go
419 lines (392 loc) · 13.1 KB
/
etag.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package etag provides an implementation of S3 ETags.
//
// Each S3 object has an associated ETag that can be
// used to e.g. quickly compare objects or check whether
// the content of an object has changed.
//
// In general, an S3 ETag is an MD5 checksum of the object
// content. However, there are many exceptions to this rule.
//
// # Single-part Upload
//
// In case of a basic single-part PUT operation - without server
// side encryption or object compression - the ETag of an object
// is its content MD5.
//
// # Multi-part Upload
//
// The ETag of an object does not correspond to its content MD5
// when the object is uploaded in multiple parts via the S3
// multipart API. Instead, S3 first computes a MD5 of each part:
//
// e1 := MD5(part-1)
// e2 := MD5(part-2)
// ...
// eN := MD5(part-N)
//
// Then, the ETag of the object is computed as MD5 of all individual
// part checksums. S3 also encodes the number of parts into the ETag
// by appending a -<number-of-parts> at the end:
//
// ETag := MD5(e1 || e2 || e3 ... || eN) || -N
//
// For example: ceb8853ddc5086cc4ab9e149f8f09c88-5
//
// However, this scheme is only used for multipart objects that are
// not encrypted.
//
// # Server-side Encryption
//
// S3 specifies three types of server-side-encryption - SSE-C, SSE-S3
// and SSE-KMS - with different semantics w.r.t. ETags.
// In case of SSE-S3, the ETag of an object is computed the same as
// for single resp. multipart plaintext objects. In particular,
// the ETag of a singlepart SSE-S3 object is its content MD5.
//
// In case of SSE-C and SSE-KMS, the ETag of an object is computed
// differently. For singlepart uploads the ETag is not the content
// MD5 of the object. For multipart uploads the ETag is also not
// the MD5 of the individual part checksums but it still contains
// the number of parts as suffix.
//
// Instead, the ETag is kind of unpredictable for S3 clients when
// an object is encrypted using SSE-C or SSE-KMS. Maybe AWS S3
// computes the ETag as MD5 of the encrypted content but there is
// no way to verify this assumption since the encryption happens
// inside AWS S3.
// Therefore, S3 clients must not make any assumption about ETags
// in case of SSE-C or SSE-KMS except that the ETag is well-formed.
//
// To put all of this into a simple rule:
//
// SSE-S3 : ETag == MD5
// SSE-C : ETag != MD5
// SSE-KMS: ETag != MD5
//
// # Encrypted ETags
//
// An S3 implementation has to remember the content MD5 of objects
// in case of SSE-S3. However, storing the ETag of an encrypted
// object in plaintext may reveal some information about the object.
// For example, two objects with the same ETag are identical with
// a very high probability.
//
// Therefore, an S3 implementation may encrypt an ETag before storing
// it. In this case, the stored ETag may not be a well-formed S3 ETag.
// For example, it can be larger due to a checksum added by authenticated
// encryption schemes. Such an ETag must be decrypted before sent to an
// S3 client.
//
// # S3 Clients
//
// There are many different S3 client implementations. Most of them
// access the ETag by looking for the HTTP response header key "Etag".
// However, some of them assume that the header key has to be "ETag"
// (case-sensitive) and will fail otherwise.
// Further, some clients require that the ETag value is a double-quoted
// string. Therefore, this package provides dedicated functions for
// adding and extracing the ETag to/from HTTP headers.
package etag
import (
"bytes"
"crypto/hmac"
"crypto/md5"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"github.com/minio/sio"
"github.com/nitrictech/minio/internal/fips"
"github.com/nitrictech/minio/internal/hash/sha256"
)
// ETag is a single S3 ETag.
//
// An S3 ETag sometimes corresponds to the MD5 of
// the S3 object content. However, when an object
// is encrypted, compressed or uploaded using
// the S3 multipart API then its ETag is not
// necessarily the MD5 of the object content.
//
// For a more detailed description of S3 ETags
// take a look at the package documentation.
type ETag []byte
// String returns the string representation of the ETag.
//
// The returned string is a hex representation of the
// binary ETag with an optional '-<part-number>' suffix.
func (e ETag) String() string {
if e.IsMultipart() {
return hex.EncodeToString(e[:16]) + string(e[16:])
}
return hex.EncodeToString(e)
}
// IsEncrypted reports whether the ETag is encrypted.
func (e ETag) IsEncrypted() bool {
// An encrypted ETag must be at least 32 bytes long.
// It contains the encrypted ETag value + an authentication
// code generated by the AEAD cipher.
//
// Here is an incorrect implementation of IsEncrypted:
//
// return len(e) > 16 && !bytes.ContainsRune(e, '-')
//
// An encrypted ETag may contain some random bytes - e.g.
// and nonce value. This nonce value may contain a '-'
// just by its nature of being randomly generated.
// The above implementation would incorrectly consider
// such an ETag (with a nonce value containing a '-')
// as non-encrypted.
return len(e) >= 32 // We consider all ETags longer than 32 bytes as encrypted
}
// IsMultipart reports whether the ETag belongs to an
// object that has been uploaded using the S3 multipart
// API.
// An S3 multipart ETag has a -<part-number> suffix.
func (e ETag) IsMultipart() bool {
return len(e) > 16 && !e.IsEncrypted() && bytes.ContainsRune(e, '-')
}
// Parts returns the number of object parts that are
// referenced by this ETag. It returns 1 if the object
// has been uploaded using the S3 singlepart API.
//
// Parts may panic if the ETag is an invalid multipart
// ETag.
func (e ETag) Parts() int {
if !e.IsMultipart() {
return 1
}
n := bytes.IndexRune(e, '-')
parts, err := strconv.Atoi(string(e[n+1:]))
if err != nil {
panic(err) // malformed ETag
}
return parts
}
// Format returns an ETag that is formatted as specified
// by AWS S3.
//
// An AWS S3 ETag is 16 bytes long and, in case of a multipart
// upload, has a `-N` suffix encoding the number of object parts.
// An ETag is not AWS S3 compatible when encrypted. When sending
// an ETag back to an S3 client it has to be formatted to be
// AWS S3 compatible.
//
// Therefore, Format returns the last 16 bytes of an encrypted
// ETag.
//
// In general, a caller has to distinguish the following cases:
// - The object is a multipart object. In this case,
// Format returns the ETag unmodified.
// - The object is a SSE-KMS or SSE-C encrypted single-
// part object. In this case, Format returns the last
// 16 bytes of the encrypted ETag which will be a random
// value.
// - The object is a SSE-S3 encrypted single-part object.
// In this case, the caller has to decrypt the ETag first
// before calling Format.
// S3 clients expect that the ETag of an SSE-S3 encrypted
// single-part object is equal to the object's content MD5.
// Formatting the SSE-S3 ETag before decryption will result
// in a random-looking ETag which an S3 client will not accept.
//
// Hence, a caller has to check:
//
// if method == SSE-S3 {
// ETag, err := Decrypt(key, ETag)
// if err != nil {
// }
// }
// ETag = ETag.Format()
func (e ETag) Format() ETag {
if !e.IsEncrypted() {
return e
}
return e[len(e)-16:]
}
var _ Tagger = ETag{} // compiler check
// ETag returns the ETag itself.
//
// By providing this method ETag implements
// the Tagger interface.
func (e ETag) ETag() ETag { return e }
// FromContentMD5 decodes and returns the Content-MD5
// as ETag, if set. If no Content-MD5 header is set
// it returns an empty ETag and no error.
func FromContentMD5(h http.Header) (ETag, error) {
v, ok := h["Content-Md5"]
if !ok {
return nil, nil
}
if v[0] == "" {
return nil, errors.New("etag: content-md5 is set but contains no value")
}
b, err := base64.StdEncoding.Strict().DecodeString(v[0])
if err != nil {
return nil, err
}
if len(b) != md5.Size {
return nil, errors.New("etag: invalid content-md5")
}
return ETag(b), nil
}
// Multipart computes an S3 multipart ETag given a list of
// S3 singlepart ETags. It returns nil if the list of
// ETags is empty.
//
// Any encrypted or multipart ETag will be ignored and not
// used to compute the returned ETag.
func Multipart(etags ...ETag) ETag {
if len(etags) == 0 {
return nil
}
var n int64
h := md5.New()
for _, etag := range etags {
if !etag.IsMultipart() && !etag.IsEncrypted() {
h.Write(etag)
n++
}
}
etag := append(h.Sum(nil), '-')
return strconv.AppendInt(etag, n, 10)
}
// Set adds the ETag to the HTTP headers. It overwrites any
// existing ETag entry.
//
// Due to legacy S3 clients, that make incorrect assumptions
// about HTTP headers, Set should be used instead of
// http.Header.Set(...). Otherwise, some S3 clients will not
// able to extract the ETag.
func Set(etag ETag, h http.Header) {
// Some (broken) S3 clients expect the ETag header to
// literally "ETag" - not "Etag". Further, some clients
// expect an ETag in double quotes. Therefore, we set the
// ETag directly as map entry instead of using http.Header.Set
h["ETag"] = []string{`"` + etag.String() + `"`}
}
// Get extracts and parses an ETag from the given HTTP headers.
// It returns an error when the HTTP headers do not contain
// an ETag entry or when the ETag is malformed.
//
// Get only accepts AWS S3 compatible ETags - i.e. no
// encrypted ETags - and therefore is stricter than Parse.
func Get(h http.Header) (ETag, error) {
const strict = true
if v := h.Get("Etag"); v != "" {
return parse(v, strict)
}
v, ok := h["ETag"]
if !ok || len(v) == 0 {
return nil, errors.New("etag: HTTP header does not contain an ETag")
}
return parse(v[0], strict)
}
// Equal returns true if and only if the two ETags are
// identical.
func Equal(a, b ETag) bool { return bytes.Equal(a, b) }
// Decrypt decrypts the ETag with the given key.
//
// If the ETag is not encrypted, Decrypt returns
// the ETag unmodified.
func Decrypt(key []byte, etag ETag) (ETag, error) {
const HMACContext = "SSE-etag"
if !etag.IsEncrypted() {
return etag, nil
}
mac := hmac.New(sha256.New, key)
mac.Write([]byte(HMACContext))
decryptionKey := mac.Sum(nil)
plaintext := make([]byte, 0, 16)
etag, err := sio.DecryptBuffer(plaintext, etag, sio.Config{
Key: decryptionKey,
CipherSuites: fips.DARECiphers(),
})
if err != nil {
return nil, err
}
return etag, nil
}
// Parse parses s as an S3 ETag, returning the result.
// The string can be an encrypted, singlepart
// or multipart S3 ETag. It returns an error if s is
// not a valid textual representation of an ETag.
func Parse(s string) (ETag, error) {
const strict = false
return parse(s, strict)
}
// parse parse s as an S3 ETag, returning the result.
// It operates in one of two modes:
// - strict
// - non-strict
//
// In strict mode, parse only accepts ETags that
// are AWS S3 compatible. In particular, an AWS
// S3 ETag always consists of a 128 bit checksum
// value and an optional -<part-number> suffix.
// Therefore, s must have the following form in
// strict mode: <32-hex-characters>[-<integer>]
//
// In non-strict mode, parse also accepts ETags
// that are not AWS S3 compatible - e.g. encrypted
// ETags.
func parse(s string, strict bool) (ETag, error) {
// An S3 ETag may be a double-quoted string.
// Therefore, we remove double quotes at the
// start and end, if any.
if strings.HasPrefix(s, `"`) && strings.HasSuffix(s, `"`) {
s = s[1 : len(s)-1]
}
// An S3 ETag may be a multipart ETag that
// contains a '-' followed by a number.
// If the ETag does not a '-' is is either
// a singlepart or encrypted ETag.
n := strings.IndexRune(s, '-')
if n == -1 {
etag, err := hex.DecodeString(s)
if err != nil {
return nil, err
}
if strict && len(etag) != 16 { // AWS S3 ETags are always 128 bit long
return nil, fmt.Errorf("etag: invalid length %d", len(etag))
}
return ETag(etag), nil
}
prefix, suffix := s[:n], s[n:]
if len(prefix) != 32 {
return nil, fmt.Errorf("etag: invalid prefix length %d", len(prefix))
}
if len(suffix) <= 1 {
return nil, errors.New("etag: suffix is not a part number")
}
etag, err := hex.DecodeString(prefix)
if err != nil {
return nil, err
}
partNumber, err := strconv.Atoi(suffix[1:]) // suffix[0] == '-' Therefore, we start parsing at suffix[1]
if err != nil {
return nil, err
}
if strict && (partNumber == 0 || partNumber > 10000) {
return nil, fmt.Errorf("etag: invalid part number %d", partNumber)
}
return ETag(append(etag, suffix...)), nil
}