Skip to content

Commit

Permalink
{satellite,storagenode}: remove old piece-transfer-based graceful exi…
Browse files Browse the repository at this point in the history 
…t code

All satellites should now be using time-based graceful exit. We can
remove the old code.

Refs: #6484
Change-Id: Ie36d14f24617c3ce1ab19ed5ec6d35f3295ccda4
  • Loading branch information
@thepaul
thepaul committed Dec 19, 2023
1 parent c4d6ccb commit ef7bdf0
Show file tree
Hide file tree
Showing 30 changed files with 92 additions and 6,120 deletions.
94 changes: 2 additions & 92 deletions cmd/satellite/gracefulexit.go
View file
Expand Up @@ -10,8 +10,6 @@ import (
"fmt"
"io"
"os"
"strconv"
"strings"
"text/tabwriter"
"time"

Expand All @@ -22,12 +20,11 @@ import (
"storj.io/common/pb"
"storj.io/common/signing"
"storj.io/common/storj"
"storj.io/storj/satellite/gracefulexit"
"storj.io/storj/satellite/satellitedb"
)

// generateGracefulExitCSV creates a report with graceful exit data for exiting or exited nodes in a given period.
func generateGracefulExitCSV(ctx context.Context, timeBased bool, completed bool, start time.Time, end time.Time, output io.Writer) error {
func generateGracefulExitCSV(ctx context.Context, completed bool, start time.Time, end time.Time, output io.Writer) error {
db, err := satellitedb.Open(ctx, zap.L().Named("db"), reportsGracefulExitCfg.Database, satellitedb.Options{ApplicationName: "satellite-gracefulexit"})
if err != nil {
return errs.New("error connecting to master database on satellite: %+v", err)
Expand Down Expand Up @@ -67,16 +64,6 @@ func generateGracefulExitCSV(ctx context.Context, timeBased bool, completed bool
if err != nil {
return err
}
exitProgress := &gracefulexit.Progress{}
if !timeBased {
exitProgress, err = db.GracefulExit().GetProgress(ctx, id)
if gracefulexit.ErrNodeNotFound.Has(err) {
exitProgress = &gracefulexit.Progress{}
} else if err != nil {
return err
}
}

exitStatus := node.ExitStatus
exitFinished := ""
if exitStatus.ExitFinishedAt != nil {
Expand All @@ -88,7 +75,7 @@ func generateGracefulExitCSV(ctx context.Context, timeBased bool, completed bool
node.CreatedAt.Format("2006-01-02"),
exitStatus.ExitInitiatedAt.Format("2006-01-02"),
exitFinished,
strconv.FormatInt(exitProgress.BytesTransferred, 10),
"0",
}
if err := w.Write(nextRow); err != nil {
return err
Expand Down Expand Up @@ -153,83 +140,6 @@ func verifyGracefulExitReceipt(ctx context.Context, identity *identity.FullIdent
return writeVerificationMessage(true, completed.SatelliteId, completed.NodeId, completed.Completed)
}

func cleanupGEOrphanedData(ctx context.Context, before time.Time, config gracefulexit.Config) (err error) {
db, err := satellitedb.Open(ctx, zap.L().Named("db"), consistencyGECleanupCfg.Database,
satellitedb.Options{ApplicationName: "satellite-gracefulexit"})
if err != nil {
return errs.New("error connecting to master database on satellite: %+v", err)
}
defer func() {
err = errs.Combine(err, db.Close())
}()

nodesItems, err := db.GracefulExit().CountFinishedTransferQueueItemsByNode(ctx, before, config.AsOfSystemTimeInterval)
if err != nil {
return err
}

if len(nodesItems) == 0 {
fmt.Printf("There isn't any item left in the DB for nodes exited before %s\n", before.Format("2006-01-02"))
return nil
}

{ // print the nodesItems
fmt.Println(" Node ID | Num. Items ")
fmt.Println("----------------------------------------------------------------------------------------")

var totalItems int64
for id, n := range nodesItems {
sid := id.String()
// 61 is the char positions between the beginning of the line and the next
// column separator, and 24 the char positions for the second column
// length. Measuring the length of the first column value (node ID), we
// calculate how many positions to shift to start printing in the next
// column and then we tell the Printf to align the value to the right by
// 24 positions which is where the column ends and where last column char
// value should end.
fmt.Printf(fmt.Sprintf(" %%s %%%dd\n", 24+61-len(sid)), sid, n)
totalItems += n
}

fmt.Println("----------------------------------------------------------------------------------------")
fmt.Printf(" Total | %22d \n\n", totalItems)
}

_, err = fmt.Printf("Confirm that you want to delete the above items from the DB? (confirm with 'yes') ")
if err != nil {
return err
}

var confirm string
n, err := fmt.Scanln(&confirm)
if err != nil {
if n != 0 {
return err
}
// fmt.Scanln cannot handle empty input
confirm = "n"
}

if strings.ToLower(confirm) != "yes" {
fmt.Println("Aborted, NO ITEMS have been deleted")
return nil
}

queueTotal, err := db.GracefulExit().DeleteAllFinishedTransferQueueItems(ctx, before, config.AsOfSystemTimeInterval, config.TransferQueueBatchSize)
if err != nil {
fmt.Println("Error, NO ITEMS have been deleted from transfer queue")
return err
}
progressTotal, err := db.GracefulExit().DeleteFinishedExitProgress(ctx, before, config.AsOfSystemTimeInterval)
if err != nil {
fmt.Printf("Error, %d stale entries were deleted from exit progress table. More stale entries might remain.\n", progressTotal)
return err
}

fmt.Printf("%d items have been deleted from transfer queue and %d items deleted from exit progress\n", queueTotal, progressTotal)
return nil
}

func checkIDs(satelliteID storj.NodeID, providedSNID storj.NodeID, receiptSatelliteID storj.NodeID, receiptSNID storj.NodeID) error {
if satelliteID != receiptSatelliteID {
return errs.New("satellite ID (%v) does not match receipt satellite ID (%v).", satelliteID, receiptSatelliteID)
Expand Down
21 changes: 4 additions & 17 deletions cmd/satellite/main.go
View file
Expand Up @@ -378,7 +378,7 @@ var (
Database string `help:"satellite database connection string" releaseDefault:"postgres://" devDefault:"postgres://"`
Output string `help:"destination of report output" default:""`
Completed bool `help:"whether to output (initiated and completed) or (initiated and not completed)" default:"false"`
TimeBased bool `help:"whether the satellite is using time-based graceful exit (and thus, whether to include piece transfer progress in output)" default:"false"`
TimeBased bool `help:"whether the satellite is using time-based graceful exit (and thus, whether to include piece transfer progress in output)" default:"true" hidden:"true"`
}
reportsVerifyGracefulExitReceiptCfg struct {
}
Expand Down Expand Up @@ -691,7 +691,7 @@ func cmdReportsGracefulExit(cmd *cobra.Command, args []string) (err error) {

// send output to stdout
if reportsGracefulExitCfg.Output == "" {
return generateGracefulExitCSV(ctx, reportsGracefulExitCfg.TimeBased, reportsGracefulExitCfg.Completed, start, end, os.Stdout)
return generateGracefulExitCSV(ctx, reportsGracefulExitCfg.Completed, start, end, os.Stdout)
}

// send output to file
Expand All @@ -704,7 +704,7 @@ func cmdReportsGracefulExit(cmd *cobra.Command, args []string) (err error) {
err = errs.Combine(err, file.Close())
}()

return generateGracefulExitCSV(ctx, reportsGracefulExitCfg.TimeBased, reportsGracefulExitCfg.Completed, start, end, file)
return generateGracefulExitCSV(ctx, reportsGracefulExitCfg.Completed, start, end, file)
}

func cmdNodeUsage(cmd *cobra.Command, args []string) (err error) {
Expand Down Expand Up @@ -969,20 +969,7 @@ func cmdStripeCustomer(cmd *cobra.Command, args []string) (err error) {
}

func cmdConsistencyGECleanup(cmd *cobra.Command, args []string) error {
ctx, _ := process.Ctx(cmd)

if runCfg.GracefulExit.TimeBased {
return errs.New("this command is not supported with time-based graceful exit")
}
before, err := time.Parse("2006-01-02", consistencyGECleanupCfg.Before)
if err != nil {
return errs.New("before flag value isn't of the expected format. %+v", err)
}

if before.After(time.Now()) {
return errs.New("before flag value cannot be newer than the current time.")
}
return cleanupGEOrphanedData(ctx, before.UTC(), runCfg.GracefulExit)
return errs.New("this command is not supported with time-based graceful exit")
}

func cmdRestoreTrash(cmd *cobra.Command, args []string) error {
Expand Down
8 changes: 0 additions & 8 deletions monkit.lock
View file
Expand Up @@ -65,19 +65,11 @@ storj.io/storj/satellite/console."login_user_captcha_unsuccessful" Counter
storj.io/storj/satellite/console."login_user_failed_count" IntVal
storj.io/storj/satellite/contact."failed_dial" Event
storj.io/storj/satellite/contact."failed_ping_node" Event
storj.io/storj/satellite/gracefulexit."graceful_exit_fail_max_failures_percentage" Meter
storj.io/storj/satellite/gracefulexit."graceful_exit_fail_validation" Meter
storj.io/storj/satellite/gracefulexit."graceful_exit_final_bytes_transferred" IntVal
storj.io/storj/satellite/gracefulexit."graceful_exit_final_pieces_failed" IntVal
storj.io/storj/satellite/gracefulexit."graceful_exit_final_pieces_succeess" IntVal
storj.io/storj/satellite/gracefulexit."graceful_exit_init_node_age_seconds" FloatVal
storj.io/storj/satellite/gracefulexit."graceful_exit_init_node_audit_success_count" IntVal
storj.io/storj/satellite/gracefulexit."graceful_exit_init_node_audit_total_count" IntVal
storj.io/storj/satellite/gracefulexit."graceful_exit_init_node_piece_count" IntVal
storj.io/storj/satellite/gracefulexit."graceful_exit_success" Meter
storj.io/storj/satellite/gracefulexit."graceful_exit_successful_pieces_transfer_ratio" IntVal
storj.io/storj/satellite/gracefulexit."graceful_exit_transfer_piece_fail" Meter
storj.io/storj/satellite/gracefulexit."graceful_exit_transfer_piece_success" Meter
storj.io/storj/satellite/metabase/rangedloop."rangedloop_error" Event
storj.io/storj/satellite/metainfo."metainfo_rate_limit_exceeded" Event
storj.io/storj/satellite/metrics."total_inline_bytes" IntVal
Expand Down
1 change: 0 additions & 1 deletion satellite/api.go
View file
Expand Up @@ -661,7 +661,6 @@ func NewAPI(log *zap.Logger, full *identity.FullIdentity, db DB,
peer.GracefulExit.Endpoint = gracefulexit.NewEndpoint(
peer.Log.Named("gracefulexit:endpoint"),
signing.SignerFromFullIdentity(peer.Identity),
peer.DB.GracefulExit(),
peer.Overlay.DB,
peer.Overlay.Service,
peer.Reputation.Service,
Expand Down

0 comments on commit ef7bdf0

Please sign in to comment.