/
trash_filewalker.go
107 lines (87 loc) · 2.99 KB
/
trash_filewalker.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
// Copyright (C) 2024 Storj Labs, Inc.
// See LICENSE for copying information.
package internalcmd
import (
"encoding/json"
"runtime"
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/common/process"
"storj.io/storj/storagenode/iopriority"
"storj.io/storj/storagenode/pieces"
"storj.io/storj/storagenode/pieces/lazyfilewalker"
"storj.io/storj/storagenode/storagenodedb"
)
// NewTrashFilewalkerCmd creates a new cobra command for running a trash cleanup filewalker.
func NewTrashFilewalkerCmd() *LazyFilewalkerCmd {
var cfg FilewalkerCfg
var runOpts RunOptions
cmd := &cobra.Command{
Use: lazyfilewalker.TrashCleanupFilewalkerCmdName,
Short: "An internal subcommand used to run a trash cleanup filewalker as a separate subprocess with lower IO priority",
RunE: func(cmd *cobra.Command, args []string) error {
runOpts.normalize(cmd)
runOpts.config = &cfg
return trashCmdRun(&runOpts)
},
FParseErrWhitelist: cobra.FParseErrWhitelist{
UnknownFlags: true,
},
Hidden: true,
Args: cobra.ExactArgs(0),
}
process.Bind(cmd, &cfg)
return NewLazyFilewalkerCmd(cmd, &runOpts)
}
// trashCmdRun runs the TrashLazyFileWalker.
func trashCmdRun(opts *RunOptions) (err error) {
if opts.config.LowerIOPriority {
if runtime.GOOS == "linux" {
// Pin the current goroutine to the current OS thread, so we can set the IO priority
// for the current thread.
// This is necessary because Go does use CLONE_IO when creating new threads,
// so they do not share a single IO context.
runtime.LockOSThread()
defer runtime.UnlockOSThread()
}
err = iopriority.SetLowIOPriority()
if err != nil {
return err
}
}
log := opts.Logger
// Decode the data struct received from the main process
var req lazyfilewalker.TrashCleanupRequest
if err = json.NewDecoder(opts.stdin).Decode(&req); err != nil {
return errs.New("Error decoding data from stdin: %v", err)
}
// Validate the request data
switch {
case req.SatelliteID.IsZero():
return errs.New("SatelliteID is required")
case req.DateBefore.IsZero():
return errs.New("DateBefore is required")
}
log.Info("trash-filewalker started", zap.Time("dateBefore", req.DateBefore))
db, err := storagenodedb.OpenExisting(opts.Ctx, log.Named("db"), opts.config.DatabaseConfig())
if err != nil {
return errs.New("Error starting master database on storage node: %v", err)
}
log.Info("Database started")
defer func() {
err = errs.Combine(err, db.Close())
}()
filewalker := pieces.NewFileWalker(log, db.Pieces(), db.V0PieceInfo())
bytesDeleted, keysDeleted, err := filewalker.WalkCleanupTrash(opts.Ctx, req.SatelliteID, req.DateBefore)
if err != nil {
return err
}
resp := lazyfilewalker.TrashCleanupResponse{
BytesDeleted: bytesDeleted,
KeysDeleted: keysDeleted,
}
log.Info("trash-filewalker completed", zap.Int64("bytesDeleted", bytesDeleted), zap.Int("numKeysDeleted", len(keysDeleted)))
// encode the response struct and write it to stdout
return json.NewEncoder(opts.stdout).Encode(resp)
}