Add option to enable sparse indexes (#3536)

configuration/configuration.go

@@ -181,25 +181,7 @@ type Configuration struct {
 	Proxy Proxy `yaml:"proxy,omitempty"`
 
 	// Validation configures validation options for the registry.
-	Validation struct {
-		// Enabled enables the other options in this section. This field is
-		// deprecated in favor of Disabled.
-		Enabled bool `yaml:"enabled,omitempty"`
-		// Disabled disables the other options in this section.
-		Disabled bool `yaml:"disabled,omitempty"`
-		// Manifests configures manifest validation.
-		Manifests struct {
-			// URLs configures validation for URLs in pushed manifests.
-			URLs struct {
-				// Allow specifies regular expressions (https://godoc.org/regexp/syntax)
-				// that URLs in pushed manifests must match.
-				Allow []string `yaml:"allow,omitempty"`
-				// Deny specifies regular expressions (https://godoc.org/regexp/syntax)
-				// that URLs in pushed manifests must not match.
-				Deny []string `yaml:"deny,omitempty"`
-			} `yaml:"urls,omitempty"`
-		} `yaml:"manifests,omitempty"`
-	} `yaml:"validation,omitempty"`
+	Validation Validation `yaml:"validation,omitempty"`
 
 	// Policy configures registry policy options.
 	Policy struct {
@@ -366,6 +348,13 @@ type Health struct {
 	} `yaml:"storagedriver,omitempty"`
 }
 
+type Platform struct {
+	// Architecture is the architecture for this platform
+	Architecture string `yaml:"architecture,omitempty"`
+	// OS is the operating system for this platform
+	OS string `yaml:"os,omitempty"`
+}
+
 // v0_1Configuration is a Version 0.1 Configuration struct
 // This is currently aliased to Configuration, as it is the current version
 type v0_1Configuration Configuration
@@ -653,6 +642,62 @@ type Proxy struct {
 	TTL *time.Duration `yaml:"ttl,omitempty"`
 }
 
+type Validation struct {
+	// Enabled enables the other options in this section. This field is
+	// deprecated in favor of Disabled.
+	Enabled bool `yaml:"enabled,omitempty"`
+	// Disabled disables the other options in this section.
+	Disabled bool `yaml:"disabled,omitempty"`
+	// Manifests configures manifest validation.
+	Manifests ValidationManifests `yaml:"manifests,omitempty"`
+}
+
+type ValidationManifests struct {
+	// URLs configures validation for URLs in pushed manifests.
+	URLs struct {
+		// Allow specifies regular expressions (https://godoc.org/regexp/syntax)
+		// that URLs in pushed manifests must match.
+		Allow []string `yaml:"allow,omitempty"`
+		// Deny specifies regular expressions (https://godoc.org/regexp/syntax)
+		// that URLs in pushed manifests must not match.
+		Deny []string `yaml:"deny,omitempty"`
+	} `yaml:"urls,omitempty"`
+	// ImageIndexes configures validation of image indexes
+	Indexes ValidationIndexes `yaml:"indexes,omitempty"`
+}
+
+type ValidationIndexes struct {
+	// Platforms configures the validation applies to the platform images included in an image index
+	Platforms Platforms `yaml:"platforms"`
+	// PlatformList filters the set of platforms to validate for image existence.
+	PlatformList []Platform `yaml:"platformlist,omitempty"`
+}
+
+// Platforms configures the validation applies to the platform images included in an image index
+// This can be all, none, or list
+type Platforms string
+
+// UnmarshalYAML implements the yaml.Umarshaler interface
+// Unmarshals a string into a Platforms option, lowercasing the string and validating that it represents a
+// valid option
+func (platforms *Platforms) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var platformsString string
+	err := unmarshal(&platformsString)
+	if err != nil {
+		return err
+	}
+
+	platformsString = strings.ToLower(platformsString)
+	switch platformsString {
+	case "all", "none", "list":
+	default:
+		return fmt.Errorf("invalid platforms option %s Must be one of [all, none, list]", platformsString)
+	}
+
+	*platforms = Platforms(platformsString)
+	return nil
+}
+
 // Parse parses an input configuration yaml document into a Configuration struct
 // This should generally be capable of handling old configuration format versions
 //

configuration/configuration_test.go

@@ -151,6 +151,13 @@ var configStruct = Configuration{
 		ReadTimeout:  time.Millisecond * 10,
 		WriteTimeout: time.Millisecond * 10,
 	},
+	Validation: Validation{
+		Manifests: ValidationManifests{
+			Indexes: ValidationIndexes{
+				Platforms: "none",
+			},
+		},
+	},
 }
 
 // configYamlV0_1 is a Version 0.1 yaml document representing configStruct
@@ -206,6 +213,10 @@ redis:
   dialtimeout: 10ms
   readtimeout: 10ms
   writetimeout: 10ms
+validation:
+  manifests:
+    indexes:
+      platforms: none
 `
 
 // inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory
@@ -235,6 +246,10 @@ notifications:
 http:
   headers:
     X-Content-Type-Options: [nosniff]
+validation:
+  manifests:
+    indexes:
+      platforms: none
 `
 
 type ConfigSuite struct {
@@ -295,6 +310,7 @@ func (suite *ConfigSuite) TestParseIncomplete() {
 	suite.expectedConfig.Notifications = Notifications{}
 	suite.expectedConfig.HTTP.Headers = nil
 	suite.expectedConfig.Redis = Redis{}
+	suite.expectedConfig.Validation.Manifests.Indexes.Platforms = ""
 
 	// Note: this also tests that REGISTRY_STORAGE and
 	// REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY can be used together
@@ -566,5 +582,11 @@ func copyConfig(config Configuration) *Configuration {
 
 	configCopy.Redis = config.Redis
 
+	configCopy.Validation = Validation{
+		Enabled:   config.Validation.Enabled,
+		Disabled:  config.Validation.Disabled,
+		Manifests: config.Validation.Manifests,
+	}
+
 	return configCopy
 }

docs/content/about/configuration.md

@@ -288,6 +288,11 @@ validation:
         - ^https?://([^/]+\.)*example\.com/
       deny:
         - ^https?://www\.example\.com/
+    indexes:
+      platforms: List
+      platformlist:
+      - architecture: amd64
+        os: linux
 ```
 
 In some instances a configuration option is **optional** but it contains child
@@ -1160,14 +1165,14 @@ username (such as `batman`) and the password for that username.
 
 ```yaml
 validation:
-  manifests:
-    urls:
-      allow:
-        - ^https?://([^/]+\.)*example\.com/
-      deny:
-        - ^https?://www\.example\.com/
+  disabled: false
 ```
 
+Use these settings to configure what validation the registry performs on content.
+
+Validation is performed when content is uploaded to the registry. Changing these
+settings will not validate content that has already been accepting into the registry.
+
 ### `disabled`
 
 The `disabled` flag disables the other options in the `validation`
@@ -1180,6 +1185,16 @@ Use the `manifests` subsection to configure validation of manifests. If
 
 #### `urls`
 
+```yaml
+validation:
+  manifests:
+    urls:
+      allow:
+        - ^https?://([^/]+\.)*example\.com/
+      deny:
+        - ^https?://www\.example\.com/
+```
+
 The `allow` and `deny` options are each a list of
 [regular expressions](https://pkg.go.dev/regexp/syntax) that restrict the URLs in
 pushed manifests.
@@ -1193,6 +1208,54 @@ one of the `allow` regular expressions **and** one of the following holds:
 2. `deny` is set but no URLs within the manifest match any of the `deny` regular
    expressions.
 
+#### `indexes`
+
+By default the registry will validate that all platform images exist when an image
+index is uploaded to the registry. Disabling this validatation is experimental
+because other tooling that uses the registry may expect the image index to be complete.
+
+validation:
+  manifests:
+    indexes:
+      platforms: [all|none|list]
+      platformlist:
+      - os: linux
+        architecture: amd64
+
+Use these settings to configure what validation the registry performs on image
+index manifests uploaded to the registry.
+
+##### `platforms`
+
+Set `platformexist` to `all` (the default) to validate all platform images exist.
+The registry will validate that the images referenced by the index exist in the
+registry before accepting the image index.
+
+Set `platforms` to `none` to disable all validation that images exist when an
+image index manifest is uploaded. This allows image lists to be uploaded to the
+registry without their associated images. This setting is experimental because
+other tooling that uses the registry may expect the image index to be complete.
+
+Set `platforms` to `list` to selectively validate the existence of platforms
+within image index manifests. This setting is experimental because other tooling
+that uses the registry may expect the image index to be complete.
+
+##### `platformlist`
+
+When `platforms` is set to `list`, set `platformlist` to an array of
+platforms to validate. If a platform is included in this the array and in the images
+contained within an index, the registry will validate that the platform specific image
+exists in the registry before accepting the index. The registry will not validate the
+existence of platform specific images in the index that do not appear in the
+`platformlist` array.
+
+This parameter does not validate that the configured platforms are included in every
+index. If an image index does not include one of the platform specific images configured
+in the `platformlist` array, it may still be accepted by the registry.
+
+Each platform is a map with two keys, `os` and `architecture`, as defined in the
+[OCI Image Index specification](https://github.com/opencontainers/image-spec/blob/main/image-index.md#image-index-property-descriptions).
+
 ## Example: Development configuration
 
 You can use this simple example for local development:

manifests.go

@@ -47,7 +47,7 @@ type ManifestBuilder interface {
 	AppendReference(dependency Describable) error
 }
 
-// ManifestService describes operations on image manifests.
+// ManifestService describes operations on manifests.
 type ManifestService interface {
 	// Exists returns true if the manifest exists.
 	Exists(ctx context.Context, dgst digest.Digest) (bool, error)

registry/handlers/api_test.go

@@ -2514,7 +2514,7 @@ func pushChunk(t *testing.T, ub *v2.URLBuilder, name reference.Named, uploadURLB
 
 func checkResponse(t *testing.T, msg string, resp *http.Response, expectedStatus int) {
 	if resp.StatusCode != expectedStatus {
-		t.Logf("unexpected status %s: %v != %v", msg, resp.StatusCode, expectedStatus)
+		t.Logf("unexpected status %s: expected %v, got %v", msg, resp.StatusCode, expectedStatus)
 		maybeDumpResponse(t, resp)
 		t.FailNow()
 	}

registry/handlers/app.go

@@ -255,6 +255,21 @@ func NewApp(ctx context.Context, config *configuration.Configuration) *App {
 				options = append(options, storage.ManifestURLsDenyRegexp(re))
 			}
 		}
+
+		switch config.Validation.Manifests.Indexes.Platforms {
+		case "list":
+			options = append(options, storage.EnableValidateImageIndexImagesExist)
+			for _, platform := range config.Validation.Manifests.Indexes.PlatformList {
+				options = append(options, storage.AddValidateImageIndexImagesExistPlatform(platform.Architecture, platform.OS))
+			}
+			fallthrough
+		case "none":
+			dcontext.GetLogger(app).Warn("Image index completeness validation has been disabled, which is an experimental option because other container tooling might expect all image indexes to be complete")
+		case "all":
+			fallthrough
+		default:
+			options = append(options, storage.EnableValidateImageIndexImagesExist)
+		}
 	}
 
 	// configure storage caches

registry/storage/manifestlisthandler.go

@@ -13,9 +13,10 @@ import (
 
 // manifestListHandler is a ManifestHandler that covers schema2 manifest lists.
 type manifestListHandler struct {
-	repository distribution.Repository
-	blobStore  distribution.BlobStore
-	ctx        context.Context
+	repository           distribution.Repository
+	blobStore            distribution.BlobStore
+	ctx                  context.Context
+	validateImageIndexes validateImageIndexes
 }
 
 var _ ManifestHandler = &manifestListHandler{}
@@ -74,24 +75,24 @@ func (ms *manifestListHandler) Put(ctx context.Context, manifestList distributio
 func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst distribution.Manifest, skipDependencyVerification bool) error {
 	var errs distribution.ErrManifestVerification
 
-	if !skipDependencyVerification {
-		// This manifest service is different from the blob service
-		// returned by Blob. It uses a linked blob store to ensure that
-		// only manifests are accessible.
-
+	// Check if we should be validating the existence of any child images in images indexes
+	if ms.validateImageIndexes.imagesExist && !skipDependencyVerification {
+		// Get the manifest service we can use to check for the existence of child images
 		manifestService, err := ms.repository.Manifests(ctx)
 		if err != nil {
 			return err
 		}
 
 		for _, manifestDescriptor := range mnfst.References() {
-			exists, err := manifestService.Exists(ctx, manifestDescriptor.Digest)
-			if err != nil && err != distribution.ErrBlobUnknown {
-				errs = append(errs, err)
-			}
-			if err != nil || !exists {
-				// On error here, we always append unknown blob errors.
-				errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: manifestDescriptor.Digest})
+			if ms.platformMustExist(manifestDescriptor) {
+				exists, err := manifestService.Exists(ctx, manifestDescriptor.Digest)
+				if err != nil && err != distribution.ErrBlobUnknown {
+					errs = append(errs, err)
+				}
+				if err != nil || !exists {
+					// On error here, we always append unknown blob errors.
+					errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: manifestDescriptor.Digest})
+				}
 			}
 		}
 	}
@@ -101,3 +102,24 @@ func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst distrib
 
 	return nil
 }
+
+// platformMustExist checks if a descriptor within an index should be validated as existing before accepting the manifest into the registry.
+func (ms *manifestListHandler) platformMustExist(descriptor distribution.Descriptor) bool {
+	// If there are no image platforms configured to validate, we must check the existence of all child images.
+	if len(ms.validateImageIndexes.imagePlatforms) == 0 {
+		return true
+	}
+
+	imagePlatform := descriptor.Platform
+
+	// If the platform matches a platform that is configured to validate, we must check the existence.
+	for _, platform := range ms.validateImageIndexes.imagePlatforms {
+		if imagePlatform.Architecture == platform.architecture &&
+			imagePlatform.OS == platform.os {
+			return true
+		}
+	}
+
+	// If the platform doesn't match a platform configured to validate, we don't need to check the existence.
+	return false
+}