Allow optionally to disable range caching.

The default behavior is to cache each range requested
to cache drive. Add a environment variable
`MINIO_RANGE_CACHE` - when set to off, it disables
range caching and instead downloads entire object
in the background.

Fixes #9870

cmd/config/cache/config.go

@@ -37,6 +37,7 @@ type Config struct {
 	After         int      `json:"after"`
 	WatermarkLow  int      `json:"watermark_low"`
 	WatermarkHigh int      `json:"watermark_high"`
+	Range         bool     `json:"range"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling

cmd/config/cache/lookup.go

@@ -34,6 +34,7 @@ const (
 	After         = "after"
 	WatermarkLow  = "watermark_low"
 	WatermarkHigh = "watermark_high"
+	Range         = "range"
 
 	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
 	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
@@ -43,6 +44,7 @@ const (
 	EnvCacheAfter         = "MINIO_CACHE_AFTER"
 	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
 	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
+	EnvCacheRange         = "MINIO_CACHE_RANGE"
 
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
 
@@ -84,6 +86,10 @@ var (
 			Key:   WatermarkHigh,
 			Value: DefaultWaterMarkHigh,
 		},
+		config.KV{
+			Key:   Range,
+			Value: config.EnableOn,
+		},
 	}
 )
 
@@ -195,5 +201,15 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		err := errors.New("config high watermark value should be greater than low watermark value")
 		return cfg, config.ErrInvalidCacheWatermarkHigh(err)
 	}
+
+	cfg.Range = true // by default range caching is enabled.
+	if rangeStr := env.Get(EnvCacheRange, kvs.Get(Range)); rangeStr != "" {
+		rng, err := config.ParseBool(rangeStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheRange(err)
+		}
+		cfg.Range = rng
+	}
+
 	return cfg, nil
 }

cmd/config/errors.go

@@ -96,6 +96,12 @@ var (
 		"MINIO_CACHE_ENCRYPTION_MASTER_KEY: For more information, please refer to https://docs.min.io/docs/minio-disk-cache-guide",
 	)
 
+	ErrInvalidCacheRange = newErrFn(
+		"Invalid cache range value",
+		"Please check the passed value",
+		"MINIO_CACHE_RANGE: Valid cache range is `on` or `off`",
+	)
+
 	ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
 		"Invalid rotating credentials",
 		"Please set correct rotating credentials in the environment for decryption",

cmd/disk-cache-backend.go

@@ -137,14 +137,15 @@ type diskCache struct {
 	after         int // minimum accesses before an object is cached.
 	lowWatermark  int
 	highWatermark int
+	enableRange   bool
 	// nsMutex namespace lock
 	nsMutex *nsLockMap
 	// Object functions pointing to the corresponding functions of backend implementation.
 	NewNSLockFn func(ctx context.Context, cachePath string) RWLocker
 }
 
 // Inits the disk cache dir if it is not initialized already.
-func newDiskCache(ctx context.Context, dir string, quotaPct, after, lowWatermark, highWatermark int) (*diskCache, error) {
+func newDiskCache(ctx context.Context, dir string, quotaPct, after, lowWatermark, highWatermark int, enableRange bool) (*diskCache, error) {
 	if err := os.MkdirAll(dir, 0777); err != nil {
 		return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
 	}
@@ -156,6 +157,7 @@ func newDiskCache(ctx context.Context, dir string, quotaPct, after, lowWatermark
 		after:         after,
 		lowWatermark:  lowWatermark,
 		highWatermark: highWatermark,
+		enableRange:   enableRange,
 		online:        1,
 		pool: sync.Pool{
 			New: func() interface{} {

cmd/disk-cache.go

@@ -312,6 +312,10 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 
 	if rs != nil {
 		go func() {
+			// if range caching is disabled, download entire object.
+			if !dcache.enableRange {
+				rs = nil
+			}
 			// fill cache in the background for range GET requests
 			bReader, bErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 			if bErr != nil {
@@ -321,7 +325,8 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			oi, _, _, err := dcache.statRange(ctx, bucket, object, rs)
 			// avoid cache overwrite if another background routine filled cache
 			if err != nil || oi.ETag != bReader.ObjInfo.ETag {
-				dcache.Put(ctx, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
+				dcache.Put(context.Background(), bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
+				return
 			}
 		}()
 		return bkReader, bkErr
@@ -546,7 +551,7 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 		if quota == 0 {
 			quota = config.Quota
 		}
-		cache, err := newDiskCache(ctx, dir, quota, config.After, config.WatermarkLow, config.WatermarkHigh)
+		cache, err := newDiskCache(ctx, dir, quota, config.After, config.WatermarkLow, config.WatermarkHigh, config.Range)
 		if err != nil {
 			return nil, false, err
 		}

docs/disk-caching/DESIGN.md

@@ -15,6 +15,7 @@ minio gateway <name> -h
      MINIO_CACHE_AFTER: Minimum number of access before caching an object.
      MINIO_CACHE_WATERMARK_LOW: % of cache quota at which cache eviction stops
      MINIO_CACHE_WATERMARK_HIGH: % of cache quota at which cache eviction starts
+     MINIO_CACHE_RANGE: Disable range caching if set to `off`.Default is `on`
 
 
 ...
@@ -62,6 +63,7 @@ Disk caching caches objects for **downloaded** objects i.e
 - When an object is deleted, corresponding entry in cache if any is deleted as well.
 - Cache continues to work for read-only operations such as GET, HEAD when backend is offline.
 - Cache-Control and Expires headers can be used to control how long objects stay in the cache. ETag of cached objects are not validated with backend until expiry time as per the Cache-Control or Expires header is met.
+- For range GET requests, each range is cached to disk by default. If you wish to download the entire object in the background to optimize future fetches, the `MINIO_CACHE_RANGE` environment variable can be set to `off` to disable range caching.
 - To ensure security guarantees, encrypted objects are normally not cached. However, if you wish to encrypt cached content on disk, you can set MINIO_CACHE_ENCRYPTION_MASTER_KEY environment variable to set a cache KMS
 master key to automatically encrypt all cached content.