Skip to content

Commit

Permalink
use nanosecond precision for timestamp in compacted boltdb-shipper in…
Browse files Browse the repository at this point in the history
…dex file names (#11277)

**What this PR does / why we need it**:
In PR #9884, we split the compaction and retention loop to run them
concurrently. Although we make sure we do not work on the same index
from compaction and retention loop, there is a chance that one could run
immediately after the other and finish quickly enough to build the index
with the same name as the previous one because in `boltdb-shipper`
index, we use epoch with `Seconds` precision while building the name of
the compacted index file. Since compaction uploads the new file first
and then deletes the old file, if the index is built with the same name,
we end up uploading the file and deleting it afterwards.

This PR fixes the issue by using ns precision for the timestamp in the
filenames.

**Special notes for your reviewer**:
This is not a problem for TSDB since we also add a checksum to the
filenames of the index during compaction.
  • Loading branch information
sandeepsukhani committed Nov 21, 2023
1 parent 658f4f1 commit 5290849
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func (c *CompactedIndex) isEmpty() (bool, error) {
// bbolt.Compact fills the whole page by setting FillPercent to 1 which works well here since while copying the data, it receives the index entries in order.
// The storage space goes down from anywhere between 25% to 50% as per my(Sandeep) tests.
func (c *CompactedIndex) recreateCompactedDB() error {
destDB, err := openBoltdbFileWithNoSync(filepath.Join(c.workingDir, fmt.Sprint(time.Now().Unix())))
destDB, err := openBoltdbFileWithNoSync(filepath.Join(c.workingDir, fmt.Sprint(time.Now().UnixNano())))
if err != nil {
return err
}
Expand Down Expand Up @@ -178,7 +178,7 @@ func (c *CompactedIndex) ToIndexFile() (shipperindex.Index, error) {
if c.compactedFileRecreated {
fileNameFormat = "%s" + recreatedCompactedDBSuffix
}
fileName := fmt.Sprintf(fileNameFormat, shipperutil.BuildIndexFileName(c.tableName, uploaderName, fmt.Sprint(time.Now().Unix())))
fileName := fmt.Sprintf(fileNameFormat, shipperutil.BuildIndexFileName(c.tableName, uploaderName, fmt.Sprint(time.Now().UnixNano())))

idxFile := boltdb.BoltDBToIndexFile(c.compactedFile, fileName)
c.compactedFile = nil
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ func (t *tableCompactor) fetchOrCreateUserCompactedIndexSet(userID string) error
return err
}

compactedFile, err := openBoltdbFileWithNoSync(filepath.Join(userIndexSet.GetWorkingDir(), fmt.Sprint(time.Now().Unix())))
compactedFile, err := openBoltdbFileWithNoSync(filepath.Join(userIndexSet.GetWorkingDir(), fmt.Sprint(time.Now().UnixNano())))
if err != nil {
return err
}
Expand All @@ -272,7 +272,7 @@ func (t *tableCompactor) fetchOrCreateUserCompactedIndexSet(userID string) error
func (t *tableCompactor) compactUserIndexes(idxSet compactor.IndexSet) (*CompactedIndex, error) {
indexes := idxSet.ListSourceFiles()
workingDir := idxSet.GetWorkingDir()
compactedDBName := filepath.Join(workingDir, fmt.Sprint(time.Now().Unix()))
compactedDBName := filepath.Join(workingDir, fmt.Sprint(time.Now().UnixNano()))

compactedFile, err := openBoltdbFileWithNoSync(compactedDBName)
if err != nil {
Expand Down Expand Up @@ -318,7 +318,7 @@ func (t *tableCompactor) compactCommonIndexes(ctx context.Context) (*CompactedIn
indexes := idxSet.ListSourceFiles()
compactedFileIdx := compactedFileIdx(indexes)
workingDir := idxSet.GetWorkingDir()
compactedDBName := filepath.Join(workingDir, fmt.Sprint(time.Now().Unix()))
compactedDBName := filepath.Join(workingDir, fmt.Sprint(time.Now().UnixNano()))

// if we find a previously compacted file, use it as a seed file to copy other index into it
if compactedFileIdx != -1 {
Expand Down

0 comments on commit 5290849

Please sign in to comment.