Skip to content

Commit

Permalink
storagenode/pieces/lazyfilewalker: more logging to GC filewalker
Browse files Browse the repository at this point in the history
Change-Id: I0103bde93a0a0a101fa413f06ae378da366dee46
  • Loading branch information
profclems committed Apr 4, 2024
1 parent 816a386 commit 780df77
Show file tree
Hide file tree
Showing 3 changed files with 18 additions and 5 deletions.
8 changes: 7 additions & 1 deletion cmd/storagenode/internalcmd/gc_filewalker.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ func gcCmdRun(g *RunOptions) (err error) {
}
err := json.NewEncoder(g.stdout).Encode(resp)
if err != nil {
log.Debug("failed to notify main process", zap.Error(err))
return err
}
numTrashed++
Expand All @@ -129,5 +130,10 @@ func gcCmdRun(g *RunOptions) (err error) {
log.Info("gc-filewalker completed", zap.Int64("piecesCount", piecesCount), zap.Int("trashPiecesCount", len(pieceIDs)), zap.Int("piecesTrashed", numTrashed), zap.Int64("piecesSkippedCount", piecesSkippedCount))

// encode the response struct and write it to stdout
return json.NewEncoder(g.stdout).Encode(resp)
err = json.NewEncoder(g.stdout).Encode(resp)
if err != nil {
log.Debug("failed to write to stdout", zap.Error(err))
return errs.New("Error writing response to stdout: %v", err)
}
return nil
}
10 changes: 8 additions & 2 deletions storagenode/pieces/lazyfilewalker/supervisor.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ func (fw *Supervisor) WalkSatellitePiecesToTrash(ctx context.Context, satelliteI

log := fw.log.Named(GCFilewalkerCmdName).With(zap.String("satelliteID", satelliteID.String()))

err = newProcess(fw.testingGCCmd, log, fw.executable, fw.gcArgs).setStdout(newTrashHandler(trashFunc)).run(ctx, req, &resp)
err = newProcess(fw.testingGCCmd, log, fw.executable, fw.gcArgs).setStdout(newTrashHandler(log, trashFunc)).run(ctx, req, &resp)
if err != nil {
return nil, 0, 0, err
}
Expand Down Expand Up @@ -184,18 +184,22 @@ func (fw *Supervisor) WalkCleanupTrash(ctx context.Context, satelliteID storj.No
type trashHandler struct {
bytes.Buffer

log *zap.Logger
lineBuffer []byte

trashFunc func(pieceID storj.PieceID) error
}

func newTrashHandler(trashFunc func(pieceID storj.PieceID) error) *trashHandler {
func newTrashHandler(log *zap.Logger, trashFunc func(pieceID storj.PieceID) error) *trashHandler {
return &trashHandler{
log: log.Named("trash-handler"),
trashFunc: trashFunc,
}
}

func (t *trashHandler) Write(b []byte) (n int, err error) {
t.log.Debug("received data from subprocess")

n = len(b)
t.lineBuffer = append(t.lineBuffer, b...)
for {
Expand Down Expand Up @@ -226,11 +230,13 @@ func (t *trashHandler) writeLine(b []byte) (remaining []byte, err error) {
func (t *trashHandler) processTrashPiece(b []byte) error {
var resp GCFilewalkerResponse
if err := json.Unmarshal(b, &resp); err != nil {
t.log.Error("failed to unmarshal data from subprocess", zap.Error(err))
return err
}

if !resp.Completed {
for _, pieceID := range resp.PieceIDs {
t.log.Debug("trashing piece", zap.String("pieceID", pieceID.String()))
return t.trashFunc(pieceID)
}
}
Expand Down
5 changes: 3 additions & 2 deletions storagenode/pieces/lazyfilewalker/supervisor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"testing"

"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"

"storj.io/common/storj"
)
Expand Down Expand Up @@ -47,7 +48,7 @@ func TestTrashHandler_Write(t *testing.T) {
expectedFinalResponse,
}

trashHandler := newTrashHandler(trashFunc)
trashHandler := newTrashHandler(zaptest.NewLogger(t), trashFunc)

for _, output := range outputs {
err := json.NewEncoder(trashHandler).Encode(output)
Expand Down Expand Up @@ -78,7 +79,7 @@ func TestTrashHandler_Write(t *testing.T) {
return nil
}

trashHandler := newTrashHandler(trashFunc)
trashHandler := newTrashHandler(zaptest.NewLogger(t), trashFunc)

// The string slice below is a concatenation of multiple JSON outputs:
// {"pieceIDs":["<pieceID0>"]}\n
Expand Down

1 comment on commit 780df77

@storjrobot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This commit has been mentioned on Storj Community Forum (official). There might be relevant details there:

https://forum.storj.io/t/current-situation-with-garbage-collection/25711/191

Please sign in to comment.