Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Refs: #6881 Change-Id: I65d101b8a2e25edbce4ca486c0d56f41c58852c0
- Loading branch information
Showing
9 changed files
with
192 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| // Copyright (C) 2024 Storj Labs, Inc. | ||
| // See LICENSE for copying information. | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "strconv" | ||
| "time" | ||
|
|
||
| "storj.io/storj/storagenode/load" | ||
| ) | ||
|
|
||
| func showIOStatsMain(args []string, output io.Writer) error { | ||
| if len(args) < 2 || len(args) > 3 { | ||
| return fmt.Errorf("usage: %s <pid> [<delay>]", args[0]) | ||
| } | ||
| delay := time.Second | ||
| var err error | ||
| if len(args) == 3 { | ||
| delay, err = time.ParseDuration(args[2]) | ||
| if err != nil { | ||
| return fmt.Errorf("could not parse duration: %w", err) | ||
| } | ||
| } | ||
| pid, err := strconv.Atoi(args[1]) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if pid == 0 { | ||
| pid = os.Getpid() | ||
| } | ||
|
|
||
| var stats1 load.Stats | ||
| err = stats1.Get(int32(pid)) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| _, _ = fmt.Fprintf(output, "%d: %+v\n", pid, stats1) | ||
| time.Sleep(delay) | ||
|
|
||
| var stats2 load.Stats | ||
| err = stats2.Get(int32(pid)) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| _, _ = fmt.Fprintf(output, | ||
| "%d: %+v\n\n"+ | ||
| "Deltas:\n"+ | ||
| "ReadCount: %d\n"+ | ||
| "WriteCount: %d\n"+ | ||
| "ReadBytes: %d\n"+ | ||
| "WriteBytes: %d\n", | ||
| pid, | ||
| stats2, | ||
| stats2.ReadCount-stats1.ReadCount, | ||
| stats2.WriteCount-stats1.WriteCount, | ||
| stats2.ReadBytes-stats1.ReadBytes, | ||
| stats2.WriteBytes-stats1.WriteBytes) | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func main() { | ||
| err := showIOStatsMain(os.Args, os.Stdout) | ||
| if err != nil { | ||
| _, _ = fmt.Fprintln(os.Stderr, err) | ||
| os.Exit(1) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| // Copyright (C) 2024 Storj Labs, Inc. | ||
| // See LICENSE for copying information. | ||
|
|
||
| package load | ||
|
|
||
| import ( | ||
| "sync" | ||
|
|
||
| "github.com/shirou/gopsutil/v3/process" | ||
| "github.com/spacemonkeygo/monkit/v3" | ||
| "go.uber.org/zap" | ||
| ) | ||
|
|
||
| // Stats represents cumulative counts of read/write operations, read/write bytes, and time spent | ||
| // doing reads and writes on a particular block device. | ||
| type Stats struct { | ||
| ReadCount uint64 | ||
| WriteCount uint64 | ||
| ReadBytes uint64 | ||
| WriteBytes uint64 | ||
| } | ||
|
|
||
| var logStatFailure sync.Once | ||
|
|
||
| // DiskIO creates a monkit StatSource that can yield disk IO statistics about the device | ||
| // underlying the specified device. | ||
| func DiskIO(logger *zap.Logger, pid int32) monkit.StatSource { | ||
| return monkit.StatSourceFunc(func(cb func(key monkit.SeriesKey, field string, val float64)) { | ||
| var stats Stats | ||
| err := stats.Get(pid) | ||
| if err != nil { | ||
| logStatFailure.Do(func() { | ||
| logger.Debug("could not get disk i/o stats", zap.Error(err)) | ||
| }) | ||
| return | ||
| } | ||
| monkit.StatSourceFromStruct(monkit.NewSeriesKey("diskio"), &stats).Stats(cb) | ||
| }) | ||
| } | ||
|
|
||
| // Get collects I/O statistics for the current process (if implemented in gopsutil). | ||
| func (s *Stats) Get(pid int32) error { | ||
| p, err := process.NewProcess(pid) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| counters, err := p.IOCounters() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| s.ReadCount = counters.ReadCount | ||
| s.WriteCount = counters.WriteCount | ||
| s.ReadBytes = counters.ReadBytes | ||
| s.WriteBytes = counters.WriteBytes | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
beddca4There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This commit has been mentioned on Storj Community Forum (official). There might be relevant details there:
https://forum.storj.io/t/node-goes-down-restarts-every-10-15-minutes-thread-allocation-error-in-logs/26010/118
beddca4There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This commit has been mentioned on Storj Community Forum (official). There might be relevant details there:
https://forum.storj.io/t/node-overloaded-traffic-flooding/26227/46