-
Notifications
You must be signed in to change notification settings - Fork 88
/
read_fallback_blob_access.go
105 lines (93 loc) · 3.67 KB
/
read_fallback_blob_access.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
package readfallback
import (
"context"
"github.com/buildbarn/bb-storage/pkg/blobstore"
"github.com/buildbarn/bb-storage/pkg/blobstore/buffer"
"github.com/buildbarn/bb-storage/pkg/blobstore/replication"
"github.com/buildbarn/bb-storage/pkg/blobstore/slicing"
"github.com/buildbarn/bb-storage/pkg/digest"
"github.com/buildbarn/bb-storage/pkg/util"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
type readFallbackBlobAccess struct {
blobstore.BlobAccess
secondary blobstore.BlobAccess
replicator replication.BlobReplicator
}
// NewReadFallbackBlobAccess creates a decorator for BlobAccess that
// causes reads for non-existent to be forwarded to a secondary storage
// backend. Data is never written to the latter.
//
// This decorator can be used to integrate external data sets into the
// system, e.g. by combining it with ReferenceExpandingBlobAccess.
func NewReadFallbackBlobAccess(primary, secondary blobstore.BlobAccess, replicator replication.BlobReplicator) blobstore.BlobAccess {
return &readFallbackBlobAccess{
BlobAccess: primary,
secondary: secondary,
replicator: replicator,
}
}
func (ba *readFallbackBlobAccess) getBlobReplicatorSelector() replication.BlobReplicatorSelector {
replicator := ba.replicator
return func(observedErr error) (replication.BlobReplicator, error) {
if status.Code(observedErr) != codes.NotFound {
// One of the backends returned an error other than
// NOT_FOUND. Prepend the name of the backend to make
// debugging easier.
if replicator != nil {
return nil, util.StatusWrap(observedErr, "Primary")
}
return nil, util.StatusWrap(observedErr, "Secondary")
}
if replicator == nil {
// We already tried the secondary below and got another
// codes.NotFound, so just return that error.
return nil, observedErr
}
replicatorToReturn := replicator
replicator = nil
return replicatorToReturn, nil
}
}
func (ba *readFallbackBlobAccess) Get(ctx context.Context, digest digest.Digest) buffer.Buffer {
return replication.GetWithBlobReplicator(
ctx,
digest,
ba.BlobAccess,
ba.getBlobReplicatorSelector())
}
func (ba *readFallbackBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer {
return replication.GetFromCompositeWithBlobReplicator(
ctx,
parentDigest,
childDigest,
slicer,
ba.BlobAccess,
ba.getBlobReplicatorSelector())
}
func (ba *readFallbackBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) {
// Call FindMissing() on the backends sequentially, as opposed
// to calling them concurrently and merging the results. In the
// common case, the primary backend is capable of pruning most
// of the digests, making the call to the secondary backend a
// lot smaller.
missingInPrimary, err := ba.BlobAccess.FindMissing(ctx, digests)
if err != nil {
return digest.EmptySet, util.StatusWrap(err, "Primary")
}
missingInBoth, err := ba.secondary.FindMissing(ctx, missingInPrimary)
if err != nil {
return digest.EmptySet, util.StatusWrap(err, "Secondary")
}
// Replicate the blobs that are present only in the secondary
// backend to the primary backend.
presentOnlyInSecondary, _, _ := digest.GetDifferenceAndIntersection(missingInPrimary, missingInBoth)
if err := ba.replicator.ReplicateMultiple(ctx, presentOnlyInSecondary); err != nil {
if status.Code(err) == codes.NotFound {
return digest.EmptySet, util.StatusWrapWithCode(err, codes.Internal, "Backend secondary returned inconsistent results while synchronizing")
}
return digest.EmptySet, util.StatusWrap(err, "Failed to synchronize from backend secondary to backend primary")
}
return missingInBoth, nil
}