From 0c56c46523b1fea148ac732d29a30d576188621b Mon Sep 17 00:00:00 2001 From: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> Date: Thu, 27 Oct 2022 16:46:31 +0530 Subject: [PATCH] rc: Add commands to set GC Percent & Memory Limit (1.19+) Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> --- fs/rc/internal.go | 94 ++++++++++++++++++++++++++++++++++++-- lib/debug/common.go | 12 +++++ lib/debug/go1.19.go | 14 ++++++ lib/debug/go1.19_compat.go | 14 ++++++ 4 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 lib/debug/common.go create mode 100644 lib/debug/go1.19.go create mode 100644 lib/debug/go1.19_compat.go diff --git a/fs/rc/internal.go b/fs/rc/internal.go index 24e67eabb0a94..4dafbb5069418 100644 --- a/fs/rc/internal.go +++ b/fs/rc/internal.go @@ -18,6 +18,7 @@ import ( "github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/buildinfo" + "github.com/rclone/rclone/lib/debug" ) func init() { @@ -300,7 +301,6 @@ Results: }) } -// Terminates app func rcSetMutexProfileFraction(ctx context.Context, in Params) (out Params, err error) { rate, err := in.GetInt64("rate") if err != nil { @@ -336,7 +336,6 @@ Parameters: }) } -// Terminates app func rcSetBlockProfileRate(ctx context.Context, in Params) (out Params, err error) { rate, err := in.GetInt64("rate") if err != nil { @@ -346,6 +345,95 @@ func rcSetBlockProfileRate(ctx context.Context, in Params) (out Params, err erro return nil, nil } +func init() { + Add(Call{ + Path: "debug/set-soft-memory-limit", + Fn: rcSetSoftMemoryLimit, + Title: "Call runtime/debug.SetMemoryLimit for setting a soft memory limit for the runtime.", + Help: ` +SetMemoryLimit provides the runtime with a soft memory limit. + +The runtime undertakes several processes to try to respect this memory limit, including +adjustments to the frequency of garbage collections and returning memory to the underlying +system more aggressively. This limit will be respected even if GOGC=off (or, if SetGCPercent(-1) is executed). + +The input limit is provided as bytes, and includes all memory mapped, managed, and not +released by the Go runtime. Notably, it does not account for space used by the Go binary +and memory external to Go, such as memory managed by the underlying system on behalf of +the process, or memory managed by non-Go code inside the same process. +Examples of excluded memory sources include: OS kernel memory held on behalf of the process, +memory allocated by C code, and memory mapped by syscall.Mmap (because it is not managed by the Go runtime). + +A zero limit or a limit that's lower than the amount of memory used by the Go runtime may cause +the garbage collector to run nearly continuously. However, the application may still make progress. + +The memory limit is always respected by the Go runtime, so to effectively disable this behavior, +set the limit very high. math.MaxInt64 is the canonical value for disabling the limit, but values +much greater than the available memory on the underlying system work just as well. + +See https://go.dev/doc/gc-guide for a detailed guide explaining the soft memory limit in more detail, +as well as a variety of common use-cases and scenarios. + +SetMemoryLimit returns the previously set memory limit. A negative input does not adjust the limit, +and allows for retrieval of the currently set memory limit. + +Parameters: + +- mem-limit - int +`, + }) +} + +func rcSetSoftMemoryLimit(ctx context.Context, in Params) (out Params, err error) { + memLimit, err := in.GetInt64("mem-limit") + if err != nil { + return nil, err + } + oldMemLimit, err := debug.SetMemoryLimit(memLimit) + if err != nil { + return nil, err + } + out = Params{ + "existing-mem-limit": oldMemLimit, + } + return out, nil +} + +func init() { + Add(Call{ + Path: "debug/set-gc-percent", + Fn: rcSetGCPercent, + Title: "Call runtime/debug.SetGCPercent for setting the garbage collection target percentage.", + Help: ` +SetGCPercent sets the garbage collection target percentage: a collection is triggered +when the ratio of freshly allocated data to live data remaining after the previous collection +reaches this percentage. SetGCPercent returns the previous setting. The initial setting is the +value of the GOGC environment variable at startup, or 100 if the variable is not set. + +This setting may be effectively reduced in order to maintain a memory limit. +A negative percentage effectively disables garbage collection, unless the memory limit is reached. + +See https://pkg.go.dev/runtime/debug#SetMemoryLimit for more details. + +Parameters: + +- gc-percent - int +`, + }) +} + +func rcSetGCPercent(ctx context.Context, in Params) (out Params, err error) { + gcPercent, err := in.GetInt64("gc-percent") + if err != nil { + return nil, err + } + oldGCPercent := debug.SetGCPercent(int(gcPercent)) + out = Params{ + "existing-gc-percent": oldGCPercent, + } + return out, nil +} + func init() { Add(Call{ Path: "core/command", @@ -384,7 +472,7 @@ Returns: "result": "" } -OR +OR { "error": true, "result": "" diff --git a/lib/debug/common.go b/lib/debug/common.go new file mode 100644 index 0000000000000..ab0a28da92f46 --- /dev/null +++ b/lib/debug/common.go @@ -0,0 +1,12 @@ +// Package debug contains functions for dealing with runtime/debug functions across go versions +package debug + +import ( + "runtime/debug" +) + +// SetGCPercent calls the runtime/debug.SetGCPercent function to set the garbage +// collection percentage. +func SetGCPercent(percent int) int { + return debug.SetGCPercent(percent) +} diff --git a/lib/debug/go1.19.go b/lib/debug/go1.19.go new file mode 100644 index 0000000000000..1103f20259e8e --- /dev/null +++ b/lib/debug/go1.19.go @@ -0,0 +1,14 @@ +//go:build go1.19 +// +build go1.19 + +package debug + +import ( + "runtime/debug" +) + +// SetMemoryLimit calls the runtime/debug.SetMemoryLimit function to set the +// soft-memory limit. +func SetMemoryLimit(limit int64) (int64, error) { + return debug.SetMemoryLimit(limit), nil +} diff --git a/lib/debug/go1.19_compat.go b/lib/debug/go1.19_compat.go new file mode 100644 index 0000000000000..e4ccbc6955b11 --- /dev/null +++ b/lib/debug/go1.19_compat.go @@ -0,0 +1,14 @@ +//go:build !go1.19 +// +build !go1.19 + +package debug + +import ( + "fmt" + "runtime" +) + +// SetMemoryLimit is a no-op on Go version < 1.19. +func SetMemoryLimit(limit int64) (int64, error) { + return limit, fmt.Errorf("not implemented on Go version below 1.19: %s", runtime.Version()) +}