Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Upgrade Thanos #1257

Merged
merged 1 commit into from
Feb 25, 2022
Merged

Upgrade Thanos #1257

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

## Grafana Mimir - main / unreleased

* [CHANGE] Compactor: No longer upload debug meta files to object storage. #1257
* [BUGFIX] Query-frontend: fixed bad querier status code mapping with query-sharding enabled. #1227

### Mixin
Expand Down
18 changes: 16 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,13 @@ require (
github.com/sirupsen/logrus v1.8.1
github.com/spf13/afero v1.6.0
github.com/stretchr/testify v1.7.0
github.com/thanos-io/thanos v0.24.0
github.com/thanos-io/thanos v0.24.1-0.20220221173150-2315a3a60a35
github.com/uber/jaeger-client-go v2.30.0+incompatible
github.com/weaveworks/common v0.0.0-20211109170639-0684aab3d884
go.uber.org/atomic v1.9.0
go.uber.org/goleak v1.1.12
golang.org/x/crypto v0.0.0-20211202192323-5770296d904e
golang.org/x/net v0.0.0-20220105145211-5b0dc2dfae98
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11
google.golang.org/grpc v1.41.1
Expand Down Expand Up @@ -83,6 +83,17 @@ require (
github.com/armon/go-metrics v0.3.9 // indirect
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
github.com/aws/aws-sdk-go v1.42.28 // indirect
github.com/aws/aws-sdk-go-v2 v1.13.0 // indirect
github.com/aws/aws-sdk-go-v2/config v1.13.1 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.10.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.2.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.7.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.9.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.14.0 // indirect
github.com/aws/smithy-go v1.10.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect
github.com/cenkalti/backoff/v4 v4.1.1 // indirect
Expand Down Expand Up @@ -174,6 +185,7 @@ require (
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/stretchr/objx v0.2.0 // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/vimeo/galaxycache v0.0.0-20210323154928-b7e5d71c067a // indirect
github.com/weaveworks/promrus v1.2.0 // indirect
go.etcd.io/etcd v3.3.25+incompatible // indirect
go.etcd.io/etcd/api/v3 v3.5.0 // indirect
Expand Down Expand Up @@ -215,3 +227,5 @@ replace github.com/thanos-io/thanos v0.24.0 => github.com/thanos-io/thanos v0.19
// Replace memberlist with our fork which includes some fixes that haven't been
// merged upstream yet.
replace github.com/hashicorp/memberlist v0.2.4 => github.com/grafana/memberlist v0.2.5-0.20211201083710-c7bc8e9df94b

replace github.com/vimeo/galaxycache => github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e
30 changes: 28 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,8 +244,30 @@ github.com/aws/aws-sdk-go v1.42.28 h1:YJtgL7IGSN41saY4JLW08jya5nU0vGcuAeAa1OL2M6
github.com/aws/aws-sdk-go v1.42.28/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
github.com/aws/aws-sdk-go-v2 v1.13.0 h1:1XIXAfxsEmbhbj5ry3D3vX+6ZcUYvIqSm4CWWEuGZCA=
github.com/aws/aws-sdk-go-v2 v1.13.0/go.mod h1:L6+ZpqHaLbAaxsqV0L4cvxZY7QupWJB4fhkf8LXvC7w=
github.com/aws/aws-sdk-go-v2/config v1.13.1 h1:yLv8bfNoT4r+UvUKQKqRtdnvuWGMK5a82l4ru9Jvnuo=
github.com/aws/aws-sdk-go-v2/config v1.13.1/go.mod h1:Ba5Z4yL/UGbjQUzsiaN378YobhFo0MLfueXGiOsYtEs=
github.com/aws/aws-sdk-go-v2/credentials v1.8.0 h1:8Ow0WcyDesGNL0No11jcgb1JAtE+WtubqXjgxau+S0o=
github.com/aws/aws-sdk-go-v2/credentials v1.8.0/go.mod h1:gnMo58Vwx3Mu7hj1wpcG8DI0s57c9o42UQ6wgTQT5to=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.10.0 h1:NITDuUZO34mqtOwFWZiXo7yAHj7kf+XPE+EiKuCBNUI=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.10.0/go.mod h1:I6/fHT/fH460v09eg2gVrd8B/IqskhNdpcLH0WNO3QI=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.4 h1:CRiQJ4E2RhfDdqbie1ZYDo8QtIo75Mk7oTdJSfwJTMQ=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.4/go.mod h1:XHgQ7Hz2WY2GAn//UXHofLfPXWh+s62MbMOijrg12Lw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.2.0 h1:3ADoioDMOtF4uiK59vCpplpCwugEU+v4ZFD29jDL3RQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.2.0/go.mod h1:BsCSJHx5DnDXIrOcqB8KN1/B+hXLG/bi4Y6Vjcx/x9E=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.5 h1:ixotxbfTCFpqbuwFv/RcZwyzhkxPSYDYEMcj4niB5Uk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.5/go.mod h1:R3sWUqPcfXSiF/LSFJhjyJmpg9uV6yP2yv3YZZjldVI=
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.7.0 h1:4QAOB3KrvI1ApJK14sliGr3Ie2pjyvNypn/lfzDHfUw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.7.0/go.mod h1:K/qPe6AP2TGYv4l6n7c88zh9jWBDf6nHhvg1fx/EWfU=
github.com/aws/aws-sdk-go-v2/service/sso v1.9.0 h1:1qLJeQGBmNQW3mBNzK2CFmrQNmoXWrscPqsrAaU1aTA=
github.com/aws/aws-sdk-go-v2/service/sso v1.9.0/go.mod h1:vCV4glupK3tR7pw7ks7Y4jYRL86VvxS+g5qk04YeWrU=
github.com/aws/aws-sdk-go-v2/service/sts v1.14.0 h1:ksiDXhvNYg0D2/UFkLejsaz3LqpW5yjNQ8Nx9Sn2c0E=
github.com/aws/aws-sdk-go-v2/service/sts v1.14.0/go.mod h1:u0xMJKDvvfocRjiozsoZglVNXRG19043xzp3r2ivLIk=
github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
github.com/aws/smithy-go v1.10.0 h1:gsoZQMNHnX+PaghNw4ynPsyGP7aUCqx5sY2dlPQsZ0w=
github.com/aws/smithy-go v1.10.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
github.com/baidubce/bce-sdk-go v0.9.81 h1:n8KfThLG9fvGv3A+RtTt/jKhg/FPPRpo+iNnS2r+iPI=
github.com/baidubce/bce-sdk-go v0.9.81/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
Expand Down Expand Up @@ -1689,6 +1711,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4=
github.com/tencentyun/cos-go-sdk-v5 v0.7.31 h1:NujkkOKMJ3IFs1+trCwXOKRCIPQ8qI5Lxul9JkhTg6M=
github.com/tencentyun/cos-go-sdk-v5 v0.7.31/go.mod h1:4E4+bQ2gBVJcgEC9Cufwylio4mXOct2iu05WjgEBx1o=
github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng=
github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM=
github.com/thanos-io/thanos v0.8.1-0.20200109203923-552ffa4c1a0d/go.mod h1:usT/TxtJQ7DzinTt+G9kinDQmRS5sxwu0unVKZ9vdcw=
github.com/thanos-io/thanos v0.13.1-0.20200731083140-69b87607decf/go.mod h1:G8caR6G7pSDreRDvFm9wFuyjEBztmr8Ag3kBYpa/fEc=
github.com/thanos-io/thanos v0.13.1-0.20200807203500-9b578afb4763/go.mod h1:KyW0a93tsh7v4hXAwo2CVAIRYuZT1Kkf4e04gisQjAg=
Expand All @@ -1699,9 +1723,10 @@ github.com/thanos-io/thanos v0.13.1-0.20210204123931-82545cdd16fe/go.mod h1:ZLDG
github.com/thanos-io/thanos v0.13.1-0.20210224074000-659446cab117/go.mod h1:kdqFpzdkveIKpNNECVJd75RPvgsAifQgJymwCdfev1w=
github.com/thanos-io/thanos v0.13.1-0.20210226164558-03dace0a1aa1/go.mod h1:gMCy4oCteKTT7VuXVvXLTPGzzjovX1VPE5p+HgL1hyU=
github.com/thanos-io/thanos v0.13.1-0.20210401085038-d7dff0c84d17/go.mod h1:zU8KqE+6A+HksK4wiep8e/3UvCZLm+Wrw9AqZGaAm9k=
github.com/thanos-io/thanos v0.19.1-0.20211229180107-bf14049d5745 h1:1b9EQbzj+HQn0GFYiOk1+JuDkeRgJNxCNa/RM4pJA0o=
github.com/thanos-io/thanos v0.19.1-0.20211229180107-bf14049d5745/go.mod h1:AmWNlejr90etGc4Rgfzx7BVm9jKwKpS3yWq3GX6M/Tg=
github.com/thanos-io/thanos v0.22.0/go.mod h1:SZDWz3phcUcBr4MYFoPFRvl+Z9Nbi45HlwQlwSZSt+Q=
github.com/thanos-io/thanos v0.24.1-0.20220221173150-2315a3a60a35 h1:Oa6kMB3c1MGBoPEki3+40r0KZMA6rwoAmpm0Dz1xiA4=
github.com/thanos-io/thanos v0.24.1-0.20220221173150-2315a3a60a35/go.mod h1:7i7BAtkc5n9gADy6Fez3PCiVb9Wzc7DldhZ+JJJC0Eg=
github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU=
github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY=
github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
Expand Down Expand Up @@ -2045,8 +2070,9 @@ golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220105145211-5b0dc2dfae98 h1:+6WJMRLHlD7X7frgp7TUZ36RnQzSf9wVVTNakEp+nqY=
golang.org/x/net v0.0.0-20220105145211-5b0dc2dfae98/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down
4 changes: 2 additions & 2 deletions pkg/compactor/bucket_compactor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -918,7 +918,7 @@ func (f *NoCompactionMarkFilter) NoCompactMarkedBlocks() map[ulid.ULID]struct{}

// Filter finds blocks that should not be compacted, and fills f.noCompactMarkedMap. If f.removeNoCompactBlocks is true,
// blocks are also removed from metas. (Thanos version of the filter doesn't do removal).
func (f *NoCompactionMarkFilter) Filter(ctx context.Context, metas map[ulid.ULID]*metadata.Meta, synced *extprom.TxGaugeVec) error {
func (f *NoCompactionMarkFilter) Filter(ctx context.Context, metas map[ulid.ULID]*metadata.Meta, synced *extprom.TxGaugeVec, modified *extprom.TxGaugeVec) error {
noCompactMarkedMap := make(map[ulid.ULID]struct{})

// Find all no-compact markers in the storage.
Expand Down Expand Up @@ -982,7 +982,7 @@ func (f *ExcludeMarkedForDeletionFilter) DeletionMarkBlocks() map[ulid.ULID]stru

// Filter filters out blocks that are marked for deletion.
// It also builds the map returned by DeletionMarkBlocks() method.
func (f *ExcludeMarkedForDeletionFilter) Filter(ctx context.Context, metas map[ulid.ULID]*metadata.Meta, synced *extprom.TxGaugeVec) error {
func (f *ExcludeMarkedForDeletionFilter) Filter(ctx context.Context, metas map[ulid.ULID]*metadata.Meta, synced *extprom.TxGaugeVec, modified *extprom.TxGaugeVec) error {
deletionMarkMap := make(map[ulid.ULID]struct{})

// Find all markers in the storage.
Expand Down
6 changes: 3 additions & 3 deletions pkg/compactor/bucket_compactor_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ func TestSyncer_GarbageCollect_e2e(t *testing.T) {
duplicateBlocksFilter := NewShardAwareDeduplicateFilter()
metaFetcher, err := block.NewMetaFetcher(nil, 32, objstore.WithNoopInstr(bkt), "", nil, []block.MetadataFilter{
duplicateBlocksFilter,
}, nil)
})
require.NoError(t, err)

blocksMarkedForDeletion := promauto.With(nil).NewCounter(prometheus.CounterOpts{})
Expand Down Expand Up @@ -236,7 +236,7 @@ func TestGroupCompactE2E(t *testing.T) {
ignoreDeletionMarkFilter,
duplicateBlocksFilter,
noCompactMarkerFilter,
}, nil)
})
require.NoError(t, err)

blocksMarkedForDeletion := promauto.With(nil).NewCounter(prometheus.CounterOpts{})
Expand Down Expand Up @@ -536,7 +536,7 @@ func TestGarbageCollectDoesntCreateEmptyBlocksWithDeletionMarksOnly(t *testing.T
metaFetcher, err := block.NewMetaFetcher(nil, 32, objstore.WithNoopInstr(bkt), "", nil, []block.MetadataFilter{
ignoreDeletionMarkFilter,
duplicateBlocksFilter,
}, nil)
})
require.NoError(t, err)

sy, err := NewMetaSyncer(nil, nil, bkt, metaFetcher, duplicateBlocksFilter, ignoreDeletionMarkFilter, blocksMarkedForDeletion, garbageCollectedBlocks)
Expand Down
8 changes: 4 additions & 4 deletions pkg/compactor/bucket_compactor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ func TestNoCompactionMarkFilter(t *testing.T) {
}

f := NewNoCompactionMarkFilter(objstore.WithNoopInstr(bkt), false)
require.NoError(t, f.Filter(ctx, metas, synced))
require.NoError(t, f.Filter(ctx, metas, synced, nil))

require.Contains(t, metas, block1)
require.Contains(t, metas, block2)
Expand All @@ -173,7 +173,7 @@ func TestNoCompactionMarkFilter(t *testing.T) {
}

f := NewNoCompactionMarkFilter(objstore.WithNoopInstr(bkt), true)
require.NoError(t, f.Filter(ctx, metas, synced))
require.NoError(t, f.Filter(ctx, metas, synced, nil))

require.Contains(t, metas, block1)
require.NotContains(t, metas, block2) // block2 was removed from metas.
Expand All @@ -200,7 +200,7 @@ func TestNoCompactionMarkFilter(t *testing.T) {
cancel()

f := NewNoCompactionMarkFilter(objstore.WithNoopInstr(bkt), true)
require.Error(t, f.Filter(canceledCtx, metas, synced))
require.Error(t, f.Filter(canceledCtx, metas, synced, nil))

require.Contains(t, metas, block1)
require.Contains(t, metas, block2)
Expand All @@ -218,7 +218,7 @@ func TestNoCompactionMarkFilter(t *testing.T) {
}

f := NewNoCompactionMarkFilter(objstore.WithNoopInstr(bkt), true)
err := f.Filter(ctx, metas, synced)
err := f.Filter(ctx, metas, synced, nil)
require.NoError(t, err)
require.Empty(t, metas)

Expand Down
1 change: 0 additions & 1 deletion pkg/compactor/compactor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -656,7 +656,6 @@ func (c *MultitenantCompactor) compactUser(ctx context.Context, userID string) e
c.metaSyncDirForUser(userID),
reg,
fetcherFilters,
nil,
)
if err != nil {
return err
Expand Down
2 changes: 1 addition & 1 deletion pkg/compactor/label_remover_filter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func NewLabelRemoverFilter(labels []string) *LabelRemoverFilter {
}

// Filter modifies external labels of existing blocks, removing given labels from the metadata of blocks that have it.
func (f *LabelRemoverFilter) Filter(_ context.Context, metas map[ulid.ULID]*metadata.Meta, _ *extprom.TxGaugeVec) error {
func (f *LabelRemoverFilter) Filter(_ context.Context, metas map[ulid.ULID]*metadata.Meta, _ *extprom.TxGaugeVec, _ *extprom.TxGaugeVec) error {
for _, meta := range metas {
for _, l := range f.labels {
delete(meta.Thanos.Labels, l)
Expand Down
2 changes: 1 addition & 1 deletion pkg/compactor/label_remover_filter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func TestLabelRemoverFilter(t *testing.T) {
}

f := NewLabelRemoverFilter(testData.labels)
err := f.Filter(context.Background(), metas, nil)
err := f.Filter(context.Background(), metas, nil, nil)
require.NoError(t, err)
assert.Len(t, metas, len(testData.expected))

Expand Down
2 changes: 1 addition & 1 deletion pkg/compactor/shard_aware_deduplicate_filter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func NewShardAwareDeduplicateFilter() *ShardAwareDeduplicateFilter {

// Filter filters out duplicate blocks that can be formed
// from two or more overlapping blocks that fully submatches the source blocks of the older blocks.
func (f *ShardAwareDeduplicateFilter) Filter(ctx context.Context, metas map[ulid.ULID]*metadata.Meta, synced *extprom.TxGaugeVec) error {
func (f *ShardAwareDeduplicateFilter) Filter(ctx context.Context, metas map[ulid.ULID]*metadata.Meta, synced *extprom.TxGaugeVec, modified *extprom.TxGaugeVec) error {
f.duplicateIDs = f.duplicateIDs[:0]

metasByResolution := make(map[int64][]*metadata.Meta)
Expand Down
4 changes: 2 additions & 2 deletions pkg/compactor/shard_aware_deduplicate_filter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,7 @@ func TestShardAwareDeduplicateFilter_Filter(t *testing.T) {
expected[id] = m
}

require.NoError(t, f.Filter(context.Background(), metas, m.Synced))
require.NoError(t, f.Filter(context.Background(), metas, m.Synced, m.Modified))
require.Equal(t, expected, metas)
require.Equal(t, float64(inputLen-len(tcase.expected)), promtest.ToFloat64(m.Synced.WithLabelValues(duplicateMeta)))
})
Expand Down Expand Up @@ -448,7 +448,7 @@ func BenchmarkDeduplicateFilter_Filter(b *testing.B) {
b.ResetTimer()
b.Run("", func(b *testing.B) {
for n := 0; n <= b.N; n++ {
_ = dedupFilter.Filter(context.Background(), tcase, synced)
_ = dedupFilter.Filter(context.Background(), tcase, synced, nil)
require.Equal(b, 0, len(dedupFilter.DuplicateIDs()))
}
})
Expand Down
2 changes: 1 addition & 1 deletion pkg/compactor/split_merge_compactor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -602,7 +602,7 @@ func TestMultitenantCompactor_ShouldSupportSplitAndMergeCompactor(t *testing.T)
fetcherDir,
reg,
[]block.MetadataFilter{NewExcludeMarkedForDeletionFilter(userBucket)},
nil)
)
require.NoError(t, err)
metas, partials, err := fetcher.Fetch(ctx)
require.NoError(t, err)
Expand Down
1 change: 0 additions & 1 deletion pkg/querier/blocks_finder_bucket_scan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,6 @@ func (d *BucketScanBlocksFinder) createMetaFetcher(userID string) (block.Metadat
filepath.Join(d.cfg.CacheDir, userID),
userReg,
filters,
nil,
)
if err != nil {
return nil, nil, nil, err
Expand Down
3 changes: 1 addition & 2 deletions pkg/storage/tsdb/bucketindex/markers_bucket_client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,7 @@ func TestGlobalMarkersBucket_DeleteShouldDeleteGlobalMarkIfBlockMarkerDoesntExis

// Delete block marker.
err := bkt.Delete(ctx, tc.blockMarker)
require.Error(t, err)
require.True(t, bkt.IsObjNotFoundErr(err))
require.NoError(t, err)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The filesystem bucket implementation in Thanos has changed, to not treat missing files as an error.


// Ensure global one been actually deleted.
verifyPathExists(t, bkt, tc.globalMarker, false)
Expand Down
2 changes: 1 addition & 1 deletion pkg/storegateway/bucket_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ func prepareStoreWithTestBlocksForSeries(t testing.TB, dir string, bkt objstore.
metaFetcher, err := block.NewMetaFetcher(s.logger, 20, objstore.WithNoopInstr(bkt), dir, nil, []block.MetadataFilter{
block.NewTimePartitionMetaFilter(filterConf.MinTime, filterConf.MaxTime),
block.NewLabelShardedMetaFilter(relabelConfig),
}, nil)
})
assert.NoError(t, err)

store, err := NewBucketStore(
Expand Down
12 changes: 1 addition & 11 deletions pkg/storegateway/bucket_index_metadata_fetcher.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ type BucketIndexMetadataFetcher struct {
cfgProvider bucket.TenantConfigProvider
logger log.Logger
filters []block.MetadataFilter
modifiers []block.MetadataModifier
metrics *block.FetcherMetrics
}

Expand All @@ -47,7 +46,6 @@ func NewBucketIndexMetadataFetcher(
logger log.Logger,
reg prometheus.Registerer,
filters []block.MetadataFilter,
modifiers []block.MetadataModifier,
) *BucketIndexMetadataFetcher {
return &BucketIndexMetadataFetcher{
userID: userID,
Expand All @@ -56,7 +54,6 @@ func NewBucketIndexMetadataFetcher(
cfgProvider: cfgProvider,
logger: logger,
filters: filters,
modifiers: modifiers,
metrics: block.NewFetcherMetrics(reg, [][]string{{corruptedBucketIndex}, {noBucketIndex}, {minTimeExcludedMeta}}, nil),
}
}
Expand Down Expand Up @@ -121,21 +118,14 @@ func (f *BucketIndexMetadataFetcher) Fetch(ctx context.Context) (metas map[ulid.
if customFilter, ok := filter.(MetadataFilterWithBucketIndex); ok {
err = customFilter.FilterWithBucketIndex(ctx, metas, idx, f.metrics.Synced)
} else {
err = filter.Filter(ctx, metas, f.metrics.Synced)
err = filter.Filter(ctx, metas, f.metrics.Synced, f.metrics.Modified)
}

if err != nil {
return nil, nil, errors.Wrap(err, "filter metas")
}
}

for _, m := range f.modifiers {
// NOTE: modifier can update modified metric accordingly to the reason of the modification.
if err := m.Modify(ctx, metas, f.metrics.Modified); err != nil {
return nil, nil, errors.Wrap(err, "modify metas")
}
}

f.metrics.Synced.WithLabelValues(block.LoadedMeta).Set(float64(len(metas)))
f.metrics.Submit()

Expand Down
Loading