global configuration - cluster scope (major upd)

* mark additional sections as cluster-scoped:
  - checksum, auth, tcb, tco, arch, rate_limit,
    keepalivetracker, net, transport, periodic
* clarify and enforce `allow:"cluster"` in the cluster config
* restrict `transient` (in memory) updates to sections NOT tagged
  with `allow:"cluster"`
-------
* [backward compatibility]: introduce `CopyPropsOpts` control
  structure and a new option
  to ignore cluster-scope violations when applying local config-override
-------
* part two, prev. commit: 445871c

Signed-off-by: Alex Aizman <alex.aizman@gmail.com>

Author: alex-aizman

README.md

@@ -16,7 +16,7 @@ AIS consistently shows [balanced I/O distribution and linear scalability](https:
 * ✅ **HTTP-based API:** A feature-rich, native API (with user-friendly SDKs for Go and Python), and compliant [Amazon S3 API](/docs/s3compat.md) for running unmodified S3 clients.
 * ✅ **Monitoring:** Comprehensive observability with integrated Prometheus metrics, Grafana dashboards, detailed logs with configurable verbosity, and CLI-based performance tracking for complete cluster visibility and troubleshooting. See [AIStore Observability](/docs/monitoring-overview.md) for details.
 * ✅ **Chunked Objects:** High-performance chunked object representation, with independently retrievable chunks, metadata v2, and checksum-protected manifests. Supports rechunking, parallel reads, and seamless integration with [Get-Batch](/docs/get_batch.md), [blob-downloader](/docs/blob_downloader.md), and multipart uploads to supported cloud backends.
-* ✅ **Secure Redirects (cluster-key):** Configurable cryptographic signing of redirect URLs using HMAC-SHA256 with a versioned cluster key.
+* ✅ **Secure Redirects:** Configurable cryptographic signing of redirect URLs using HMAC-SHA256 with a versioned cluster key (stored in memory only).
 * ✅ **Load-Aware Throttling:** Dynamic request throttling based on a five-dimensional load vector (CPU, memory, disk, FDs, goroutines) to protect AIS clusters under stress.
 * ✅ **Unified Namespace:** Attach AIS clusters together to provide fast, unified access to the entirety of hosted datasets, allowing users to reference shared buckets with cluster-specific identifiers.
 * ✅ **Turn-key Cache:** In addition to robust data protection features, AIS offers a per-bucket configurable LRU-based cache with eviction thresholds and storage capacity watermarks.

ais/daemon.go

@@ -174,7 +174,7 @@ func initDaemon(version, buildTime string) cos.Runner {
 		if err := toUpdate.FillFromKVS(kvs); err != nil {
 			cos.ExitLog(err)
 		}
-		if err := setConfigInMem(toUpdate, config, apc.Daemon); err != nil {
+		if err := setConfigInMem(toUpdate, config, apc.Daemon, true /*transient*/); err != nil {
 			cos.ExitLogf("failed to update config in memory: %v", err)
 		}
 

ais/gconfig.go

@@ -187,11 +187,10 @@ func (*configOwner) persistBytes(payload msPayload, globalFpath string) (done bo
 	return
 }
 
-// NOTE: must be called under config-owner lock
-func setConfig(toUpdate *cmn.ConfigToSet, transient bool) (err error) {
+// must be called under config-owner lock
+func setConfig(toUpdate *cmn.ConfigToSet, transient bool) error {
 	clone := cmn.GCO.Clone()
-	err = setConfigInMem(toUpdate, clone, apc.Daemon)
-	if err != nil {
+	if err := setConfigInMem(toUpdate, clone, apc.Daemon, transient); err != nil {
 		return err
 	}
 	override := cmn.GCO.GetOverride()
@@ -211,9 +210,8 @@ func setConfig(toUpdate *cmn.ConfigToSet, transient bool) (err error) {
 	return nil
 }
 
-func setConfigInMem(toUpdate *cmn.ConfigToSet, config *cmn.Config, asType string) (err error) {
-	err = config.UpdateClusterConfig(toUpdate, asType)
-	return
+func setConfigInMem(toUpdate *cmn.ConfigToSet, config *cmn.Config, asType string, transient bool) error {
+	return config.UpdateClusterConfig(toUpdate, asType, cmn.CopyPropsOpts{Transient: transient})
 }
 
 func (co *configOwner) resetDaemonConfig() (err error) {

ais/prxclu.go

@@ -1246,7 +1246,7 @@ func (p *proxy) setCluCfgTransient(w http.ResponseWriter, r *http.Request, toUpd
 }
 
 func _setConfPre(ctx *configModifier, clone *globalConfig) (updated bool, err error) {
-	if err = clone.Apply(ctx.toUpdate, apc.Cluster); err != nil {
+	if err = cmn.CopyProps(ctx.toUpdate, clone, apc.Cluster); err != nil {
 		return
 	}
 	updated = true

cmn/config.go

@@ -31,12 +31,8 @@ import (
 	jsoniter "github.com/json-iterator/go"
 )
 
-// TODO post-4.1:
-// - revisit `allow:"cluster"` usage across ClusterConfig
-// - for 4.1, making a single change - marking `AuthConf` as cluster-scoped
-
 const (
-	confDisabled = "Disabled"
+	confDisabled = "Disabled" // common conf.String()
 )
 
 type (
@@ -98,7 +94,10 @@ type (
 	}
 )
 
-// global configuration
+// Global configuration
+// Note: updating any of these fields on a per-node basis:
+// - will fail for (cluster-scoped) sections tagged with `allow:"cluster"`
+// - is at your own risk otherwise; such changes may cause inconsistent behavior across the cluster.
 type (
 	ClusterConfig struct {
 		Backend     BackendConf     `json:"backend" allow:"cluster"`
@@ -108,29 +107,29 @@ type (
 		UUID        string          `json:"uuid"`
 		Dsort       DsortConf       `json:"distributed_sort"`
 		Proxy       ProxyConf       `json:"proxy" allow:"cluster"`
-		Cksum       CksumConf       `json:"checksum"`
+		Cksum       CksumConf       `json:"checksum" allow:"cluster"`
 		Auth        AuthConf        `json:"auth" allow:"cluster"`
 		Tracing     TracingConf     `json:"tracing"`
-		TCB         TCBConf         `json:"tcb"`
-		TCO         TCOConf         `json:"tco"`
-		Arch        ArchConf        `json:"arch"`
+		TCB         TCBConf         `json:"tcb" allow:"cluster"`
+		TCO         TCOConf         `json:"tco" allow:"cluster"`
+		Arch        ArchConf        `json:"arch" allow:"cluster"`
 		RateLimit   RateLimitConf   `json:"rate_limit"`
 		Keepalive   KeepaliveConf   `json:"keepalivetracker"`
 		Rebalance   RebalanceConf   `json:"rebalance" allow:"cluster"`
 		Log         LogConf         `json:"log"`
 		EC          ECConf          `json:"ec" allow:"cluster"`
-		Net         NetConf         `json:"net"`
+		Net         NetConf         `json:"net" allow:"cluster"`
 		Timeout     TimeoutConf     `json:"timeout"`
 		Space       SpaceConf       `json:"space"`
-		Transport   TransportConf   `json:"transport"`
+		Transport   TransportConf   `json:"transport" allow:"cluster"`
 		Memsys      MemsysConf      `json:"memsys"`
 		Disk        DiskConf        `json:"disk"`
 		FSHC        FSHCConf        `json:"fshc"`
 		Chunks      ChunksConf      `json:"chunks" allow:"cluster"`
 		LRU         LRUConf         `json:"lru"`
 		Client      ClientConf      `json:"client"`
 		Mirror      MirrorConf      `json:"mirror" allow:"cluster"`
-		Periodic    PeriodConf      `json:"periodic"`
+		Periodic    PeriodConf      `json:"periodic" allow:"cluster"`
 		Downloader  DownloaderConf  `json:"downloader"`
 		Features    feat.Flags      `json:"features,string" allow:"cluster"` // enumerated features to flip assorted global defaults (cmn/feat/feat and docs/feat*)
 		Version     int64           `json:"config_version,string"`
@@ -1065,9 +1064,8 @@ func (c *Config) SetRole(role string) {
 	c.role = role
 }
 
-func (c *Config) UpdateClusterConfig(updateConf *ConfigToSet, asType string) (err error) {
-	err = c.ClusterConfig.Apply(updateConf, asType)
-	if err != nil {
+func (c *Config) UpdateClusterConfig(updateConf *ConfigToSet, asType string, opts CopyPropsOpts) (err error) {
+	if err = CopyProps(updateConf, &c.ClusterConfig, asType, opts); err != nil {
 		return
 	}
 	return c.Validate()
@@ -1129,10 +1127,6 @@ func (ctu *ConfigToSet) FillFromKVS(kvs []string) (err error) {
 // ClusterConfig //
 ///////////////////
 
-func (c *ClusterConfig) Apply(updateConf *ConfigToSet, asType string) error {
-	return CopyProps(updateConf, c, asType)
-}
-
 func (c *ClusterConfig) String() string {
 	if c == nil {
 		return "Conf <nil>"
@@ -2782,7 +2776,7 @@ func handleOverrideConfig(config *Config) error {
 		config.LocalConfig.FSP = *overrideConfig.FSP // override local config's fspaths
 		overrideConfig.FSP = nil
 	}
-	return config.UpdateClusterConfig(overrideConfig, apc.Daemon)
+	return config.UpdateClusterConfig(overrideConfig, apc.Daemon, CopyPropsOpts{Transient: false, IgnoreScope: true})
 }
 
 func SaveOverrideConfig(configDir string, toUpdate *ConfigToSet) error {

cmn/gco.go

@@ -110,7 +110,7 @@ func (gco *gco) Update(cluConfig *ClusterConfig) (err error) {
 	config.ClusterConfig = *cluConfig
 	override := gco.GetOverride()
 	if override != nil {
-		err = config.UpdateClusterConfig(override, apc.Daemon) // update and validate
+		err = config.UpdateClusterConfig(override, apc.Daemon, CopyPropsOpts{Transient: false, IgnoreScope: true}) // update and validate
 	} else {
 		err = config.Validate()
 	}

cmn/iter_fields.go

@@ -16,6 +16,7 @@ import (
 	"github.com/NVIDIA/aistore/api/apc"
 	"github.com/NVIDIA/aistore/cmn/cos"
 	"github.com/NVIDIA/aistore/cmn/debug"
+	"github.com/NVIDIA/aistore/cmn/nlog"
 )
 
 const IterFieldNameSepa = "."
@@ -56,6 +57,14 @@ type (
 	updateFunc func(uniqueTag string, field IterField) (error, bool)
 )
 
+type (
+	// controls how CopyProps/_copyProps enforce config scope and transient rules
+	CopyPropsOpts struct {
+		Transient   bool // treat changes as transient: forbid writes to allow:"cluster"
+		IgnoreScope bool // when true, skip allow:"cluster" violations instead of failing (config override-only)
+	}
+)
+
 // interface guard
 var _ IterField = (*field)(nil)
 
@@ -233,16 +242,20 @@ func iterFields(prefix string, v any, updf updateFunc, opts IterOpts) (dirty, st
 }
 
 // CopyProps update dst with the values from src
-func CopyProps(src, dst any, asType string) error {
+func CopyProps(src, dst any, asType string, copts ...CopyPropsOpts) error {
 	var (
 		srcVal = reflect.ValueOf(src)
 		dstVal = reflect.ValueOf(dst).Elem()
+		opts   CopyPropsOpts
 	)
+	if len(copts) > 0 {
+		opts = copts[0]
+	}
 	debug.Assertf(slices.Contains([]string{apc.Daemon, apc.Cluster}, asType), "unexpected config level: %s", asType)
 	if srcVal.Kind() == reflect.Ptr {
 		srcVal = srcVal.Elem()
 	}
-	return _copyProps(srcVal, dstVal, asType)
+	return _copyProps(srcVal, dstVal, asType, opts)
 }
 
 // copyProps helper: whether v.Kind() supports IsNil()
@@ -277,7 +290,7 @@ func peel2(v reflect.Value) reflect.Value {
 	return v
 }
 
-func _copyProps(srcVal, dstVal reflect.Value, asType string) error {
+func _copyProps(srcVal, dstVal reflect.Value, asType string, opts CopyPropsOpts) error {
 	// normalize pointers on entry
 	for srcVal.Kind() == reflect.Ptr && !srcVal.IsNil() {
 		srcVal = srcVal.Elem()
@@ -353,7 +366,7 @@ func _copyProps(srcVal, dstVal reflect.Value, asType string) error {
 				if nilable(srcField.Kind()) && srcField.IsNil() {
 					continue
 				}
-				if err := _copyProps(srcField, dstVal, asType); err != nil {
+				if err := _copyProps(srcField, dstVal, asType, opts); err != nil {
 					return err
 				}
 				continue
@@ -372,7 +385,17 @@ func _copyProps(srcVal, dstVal reflect.Value, asType string) error {
 		// scope enforcement from dst tag
 		if dtf, ok := dstVal.Type().FieldByName(fieldName); ok {
 			allowedScope := dtf.Tag.Get("allow")
-			if allowedScope != "" && allowedScope != asType {
+
+			// 3 special cases
+			switch {
+			case opts.IgnoreScope && allowedScope == apc.Cluster && asType == apc.Daemon:
+				name := strings.ToLower(fieldName)
+				nlog.Warningln("ignoring node override for cluster-scoped config:", name)
+				continue
+			case opts.Transient && allowedScope == apc.Cluster:
+				name := strings.ToLower(fieldName)
+				return fmt.Errorf("%s (cluster-scoped) configuration cannot be changed transiently", name)
+			case allowedScope != "" && allowedScope != asType:
 				name := strings.ToLower(fieldName)
 				if allowedScope == apc.Cluster && asType == apc.Daemon {
 					return fmt.Errorf("%s configuration can only be globally updated", name)
@@ -401,7 +424,7 @@ func _copyProps(srcVal, dstVal reflect.Value, asType string) error {
 
 		// recurse only when _both_ are structs
 		if s.Kind() == reflect.Struct && d.Kind() == reflect.Struct {
-			if err := _copyProps(srcField, dstField, asType); err != nil { // pass originals to allow alloc
+			if err := _copyProps(srcField, dstField, asType, opts); err != nil { // pass originals to allow alloc
 				return err
 			}
 			continue