From 3fac2a702797548492c81ae566dfd1014bb0008b Mon Sep 17 00:00:00 2001 From: henrikschristensen <51989221+henrikschristensen@users.noreply.github.com> Date: Sun, 18 Feb 2024 18:50:15 +0000 Subject: [PATCH] Fixed windows compatibility issue, when storage gateway syncs index files to local disk. Signed-off-by: henrikschristensen <51989221+henrikschristensen@users.noreply.github.com> --- CHANGELOG.md | 3 ++- pkg/block/indexheader/binary_reader.go | 25 +++++++++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cddbd300efd..aac15c3802d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,10 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ## Unreleased ### Fixed - + - [#7083](https://github.com/thanos-io/thanos/pull/7083) Store Gateway: Fix lazy expanded postings with 0 length failed to be cached. - [#7080](https://github.com/thanos-io/thanos/pull/7080) Receive: race condition in handler Close() when stopped early +- [#7148](https://github.com/thanos-io/thanos/pull/7148) Store Gateway: Fix windows compatibility in index-header sync. - [#7132](https://github.com/thanos-io/thanos/pull/7132) Documentation: fix broken helm installation instruction ### Added diff --git a/pkg/block/indexheader/binary_reader.go b/pkg/block/indexheader/binary_reader.go index c86185bfbf8..56ce61188da 100644 --- a/pkg/block/indexheader/binary_reader.go +++ b/pkg/block/indexheader/binary_reader.go @@ -13,6 +13,7 @@ import ( "io" "math" "os" + "path" "path/filepath" "sort" "sync" @@ -112,6 +113,20 @@ func WriteBinary(ctx context.Context, bkt objstore.BucketReader, id ulid.ULID, f tmpFilename = filename + ".tmp" } + bw, err := writeAndClose(id, tmpFilename, indexVersion, ir) + if err != nil { + return nil, err + } + + if tmpFilename != "" { + // Create index-header in atomic way, to avoid partial writes (e.g during restart or crash of store GW). + return nil, os.Rename(tmpFilename, filename) + } + + return bw.Buffer(), nil +} + +func writeAndClose(id ulid.ULID, tmpFilename string, indexVersion int, ir *chunkedIndexReader) (*binaryWriter, error) { // Buffer for copying and encbuffers. // This also will control the size of file writer buffer. buf := make([]byte, 32*1024) @@ -152,13 +167,7 @@ func WriteBinary(ctx context.Context, bkt objstore.BucketReader, id ulid.ULID, f if err := bw.writer.Sync(); err != nil { return nil, errors.Wrap(err, "sync") } - - if tmpFilename != "" { - // Create index-header in atomic way, to avoid partial writes (e.g during restart or crash of store GW). - return nil, os.Rename(tmpFilename, filename) - } - - return bw.Buffer(), nil + return bw, nil } type chunkedIndexReader struct { @@ -170,7 +179,7 @@ type chunkedIndexReader struct { } func newChunkedIndexReader(ctx context.Context, bkt objstore.BucketReader, id ulid.ULID) (*chunkedIndexReader, int, error) { - indexFilepath := filepath.Join(id.String(), block.IndexFilename) + indexFilepath := path.Join(id.String(), block.IndexFilename) attrs, err := bkt.Attributes(ctx, indexFilepath) if err != nil { return nil, 0, errors.Wrapf(err, "get object attributes of %s", indexFilepath)