/
limiter.go
135 lines (108 loc) · 4.22 KB
/
limiter.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
package drive
import (
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/pkg/control"
)
var (
errHitLimit = clues.New("hit limiter limits")
errHitCollectionLimit = clues.New("hit item limits within the current collection")
)
type driveEnumerationStats struct {
numPages int
numAddedFiles int
numContainers int
numBytes int64
}
func newPagerLimiter(opts control.Options) *pagerLimiter {
res := &pagerLimiter{limits: opts.PreviewLimits}
if res.limits.MaxContainers == 0 {
res.limits.MaxContainers = defaultPreviewMaxContainers
}
if res.limits.MaxItemsPerContainer == 0 {
res.limits.MaxItemsPerContainer = defaultPreviewMaxItemsPerContainer
}
if res.limits.MaxItems == 0 {
res.limits.MaxItems = defaultPreviewMaxItems
}
if res.limits.MaxBytes == 0 {
res.limits.MaxBytes = defaultPreviewMaxBytes
}
if res.limits.MaxPages == 0 {
res.limits.MaxPages = defaultPreviewMaxPages
}
return res
}
type pagerLimiter struct {
limits control.PreviewItemLimits
}
func (l pagerLimiter) effectiveLimits() control.PreviewItemLimits {
return l.limits
}
func (l pagerLimiter) enabled() bool {
return l.limits.Enabled
}
// sizeLimit returns the total number of bytes this backup should try to
// contain.
func (l pagerLimiter) sizeLimit() int64 {
return l.limits.MaxBytes
}
// atItemLimit returns true if the limiter is enabled and has reached the limit
// for individual items added to collections for this backup.
func (l pagerLimiter) atItemLimit(stats *driveEnumerationStats) bool {
return l.enabled() &&
(stats.numAddedFiles >= l.limits.MaxItems ||
stats.numBytes >= l.limits.MaxBytes)
}
// atContainerItemsLimit returns true if the limiter is enabled and the current
// number of items is above the limit for the number of items for a container
// for this backup.
func (l pagerLimiter) atContainerItemsLimit(numItems int) bool {
return l.enabled() && numItems >= l.limits.MaxItemsPerContainer
}
// atPageLimit returns true if the limiter is enabled and the number of
// pages processed so far is beyond the limit for this backup.
func (l pagerLimiter) atPageLimit(stats *driveEnumerationStats) bool {
return l.enabled() && stats.numPages >= l.limits.MaxPages
}
// atLimit returns true if the limiter is enabled and meets any of the
// conditions for max items, containers, etc for this backup.
func (l pagerLimiter) atLimit(stats *driveEnumerationStats) bool {
return l.enabled() &&
(l.atItemLimit(stats) ||
stats.numContainers >= l.limits.MaxContainers ||
stats.numPages >= l.limits.MaxPages)
}
// ---------------------------------------------------------------------------
// Used by the tree version limit handling
// ---------------------------------------------------------------------------
// hitPageLimit returns true if the limiter is enabled and the number of
// pages processed so far is beyond the limit for this backup.
func (l pagerLimiter) hitPageLimit(pageCount int) bool {
return l.enabled() && pageCount >= l.limits.MaxPages
}
// hitContainerLimit returns true if the limiter is enabled and the number of
// unique containers added so far is beyond the limit for this backup.
func (l pagerLimiter) hitContainerLimit(containerCount int) bool {
return l.enabled() && containerCount >= l.limits.MaxContainers
}
// hitItemLimit returns true if the limiter is enabled and has reached the limit
// for unique items added to collections for this backup.
func (l pagerLimiter) hitItemLimit(itemCount int) bool {
return l.enabled() && itemCount >= l.limits.MaxItems
}
// alreadyHitTotalBytesLimit returns true if the limiter is enabled and has reached the limit
// for the accumulated byte size of all items (the file contents, not the item metadata)
// added to collections for this backup.
func (l pagerLimiter) alreadyHitTotalBytesLimit(i int64) bool {
return l.enabled() && i > l.limits.MaxBytes
}
// willStepOverBytesLimit returns true if the limiter is enabled and the provided addition
// of bytes is greater than the limit plus some padding (to ensure we can always hit
// the limit).
func (l pagerLimiter) willStepOverBytesLimit(current, addition int64) bool {
if !l.enabled() {
return false
}
limitPlusPadding := int64(float64(l.limits.MaxBytes) * 1.03)
return (current + addition) > limitPlusPadding
}