scorecard (alpha): use bundle labels in pod runner

cmd/operator-sdk/alpha/scorecard/cmd.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 
 	"github.com/operator-framework/operator-sdk/internal/flags"
+	registryutil "github.com/operator-framework/operator-sdk/internal/registry"
 	scorecard "github.com/operator-framework/operator-sdk/internal/scorecard/alpha"
 	"github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3"
 )
@@ -62,6 +63,7 @@ If the argument holds an image tag, it must be present remotely.`,
 			return c.validate(args)
 		},
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			c.bundle = args[0]
 			return c.run()
 		},
 	}
@@ -108,27 +110,30 @@ func (c *scorecardCmd) printOutput(output v1alpha3.Test) error {
 		return fmt.Errorf("invalid output format selected")
 	}
 	return nil
-
 }
 
-func (c *scorecardCmd) run() error {
-	var err error
+func (c *scorecardCmd) run() (err error) {
 	// Extract bundle image contents if bundle is inferred to be an image.
+	var bundleLabels registryutil.Labels
 	if _, err = os.Stat(c.bundle); err != nil && errors.Is(err, os.ErrNotExist) {
-		// Discard bundle extraction logs unless user sets verbose mode.
-		logger := log.NewEntry(discardLogger())
-		if viper.GetBool(flags.VerboseOpt) {
-			logger = log.WithFields(log.Fields{"bundle": c.bundle})
-		}
-		// FEAT: enable explicit local image extraction.
-		if c.bundle, err = scorecard.ExtractBundleImage(context.TODO(), logger, c.bundle, false); err != nil {
+		if c.bundle, bundleLabels, err = getBundlePathAndLabelsFromImage(c.bundle); err != nil {
 			log.Fatal(err)
 		}
 		defer func() {
 			if err := os.RemoveAll(c.bundle); err != nil {
-				logger.Error(err)
+				log.Error(err)
 			}
 		}()
+	} else {
+		// Search for the metadata dir since we cannot assume its path, and
+		// use that metadata as the source of truth when testing.
+		metadataDir, err := registryutil.FindMetadataDir(c.bundle)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if bundleLabels, err = registryutil.GetMetadataLabels(metadataDir); err != nil {
+			log.Fatal(err)
+		}
 	}
 
 	o := scorecard.Scorecard{
@@ -160,6 +165,7 @@ func (c *scorecardCmd) run() error {
 			ServiceAccount: c.serviceAccount,
 			Namespace:      c.namespace,
 			BundlePath:     c.bundle,
+			BundleLabels:   bundleLabels,
 		}
 
 		// Only get the client if running tests.
@@ -185,8 +191,6 @@ func (c *scorecardCmd) validate(args []string) error {
 	if len(args) != 1 {
 		return fmt.Errorf("a bundle image or directory argument is required")
 	}
-	c.bundle = args[0]
-
 	return nil
 }
 
@@ -196,3 +200,27 @@ func discardLogger() *log.Logger {
 	logger.SetOutput(ioutil.Discard)
 	return logger
 }
+
+// getBundlePathAndLabelsFromImage returns bundleImage's path on disk post-
+// extraction and image labels.
+func getBundlePathAndLabelsFromImage(bundleImage string) (string, registryutil.Labels, error) {
+	// Discard bundle extraction logs unless user sets verbose mode.
+	logger := log.NewEntry(discardLogger())
+	if viper.GetBool(flags.VerboseOpt) {
+		logger = log.WithFields(log.Fields{"bundle": bundleImage})
+	}
+	// FEAT: enable explicit local image extraction.
+	bundlePath, err := registryutil.ExtractBundleImage(context.TODO(), logger, bundleImage, false)
+	if err != nil {
+		return "", nil, err
+	}
+
+	// Get image labels from bundleImage locally, since the bundle extraction
+	// already pulled the image.
+	bundleLabels, err := registryutil.GetImageLabels(context.TODO(), logger, bundleImage, true)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return bundlePath, bundleLabels, nil
+}

cmd/operator-sdk/alpha/scorecard/cmd_test.go

@@ -81,11 +81,10 @@ var _ = Describe("Running an alpha scorecard command", func() {
 			Expect(err).To(HaveOccurred())
 		})
 
-		It("succeeds if exactly one arg is provided and parses the arg", func() {
+		It("succeeds if exactly one arg is provided", func() {
 			input := "cherry"
 			err := cmd.validate([]string{input})
 			Expect(err).NotTo(HaveOccurred())
-			Expect(cmd.bundle).To(Equal(input))
 		})
 	})
 })

images/scorecard-test/cmd/test/main.go

@@ -22,6 +22,7 @@ import (
 
 	apimanifests "github.com/operator-framework/api/pkg/manifests"
 
+	registryutil "github.com/operator-framework/operator-sdk/internal/registry"
 	scorecard "github.com/operator-framework/operator-sdk/internal/scorecard/alpha"
 	"github.com/operator-framework/operator-sdk/internal/scorecard/alpha/tests"
 	scapiv1alpha3 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3"
@@ -48,11 +49,16 @@ func main() {
 		log.Fatal(err.Error())
 	}
 
+	labels, err := registryutil.GetMetadataLabels(scorecard.PodLabelsDir)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+
 	var result scapiv1alpha3.TestStatus
 
 	switch entrypoint[0] {
 	case tests.OLMBundleValidationTest:
-		result = tests.BundleValidationTest(scorecard.PodBundleRoot)
+		result = tests.BundleValidationTest(scorecard.PodBundleRoot, labels)
 	case tests.OLMCRDsHaveValidationTest:
 		result = tests.CRDsHaveValidationTest(cfg)
 	case tests.OLMCRDsHaveResourcesTest:

internal/scorecard/alpha/registry.go → internal/registry/image.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package alpha
+package registry
 
 import (
 	"context"

internal/registry/labels.go

@@ -0,0 +1,129 @@
+// Copyright 2020 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package registry
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	registryimage "github.com/operator-framework/operator-registry/pkg/image"
+	"github.com/operator-framework/operator-registry/pkg/image/containerdregistry"
+	registrybundle "github.com/operator-framework/operator-registry/pkg/lib/bundle"
+	log "github.com/sirupsen/logrus"
+
+	// TODO: replace `gopkg.in/yaml.v2` with `sigs.k8s.io/yaml` once operator-registry has `json` tags in the
+	// annotations struct.
+	yaml "gopkg.in/yaml.v3"
+)
+
+// Labels is a set of key:value labels from an operator-registry object.
+type Labels map[string]string
+
+// GetManifestsDir returns the manifests directory name in ls using
+// a predefined key, or false if it does not exist.
+func (ls Labels) GetManifestsDir() (string, bool) {
+	value, hasLabel := ls.getLabel(registrybundle.ManifestsLabel)
+	return filepath.Clean(value), hasLabel
+}
+
+// getLabel returns the string by key in ls, or an empty string and false
+// if key is not found in ls.
+func (ls Labels) getLabel(key string) (string, bool) {
+	value, hasLabel := ls[key]
+	return value, hasLabel
+}
+
+// GetImageLabels returns the set of labels on image.
+func GetImageLabels(ctx context.Context, logger *log.Entry, image string, local bool) (Labels, error) {
+	// Create a containerd registry for socket-less image layer reading.
+	reg, err := containerdregistry.NewRegistry(containerdregistry.WithLog(logger))
+	if err != nil {
+		return nil, fmt.Errorf("error creating new image registry: %v", err)
+	}
+	defer func() {
+		if err := reg.Destroy(); err != nil {
+			logger.WithError(err).Warn("Error destroying local cache")
+		}
+	}()
+
+	// Pull the image if it isn't present locally.
+	if !local {
+		if err := reg.Pull(ctx, registryimage.SimpleReference(image)); err != nil {
+			return nil, fmt.Errorf("error pulling image %s: %v", image, err)
+		}
+	}
+
+	// Query the image reference for its labels.
+	labels, err := reg.Labels(ctx, registryimage.SimpleReference(image))
+	if err != nil {
+		return nil, fmt.Errorf("error reading image %s labels: %v", image, err)
+	}
+
+	return labels, err
+}
+
+// FindMetadataDir walks bundleRoot searching for metadata, and returns that directory if found.
+// If one is not found, an error is returned.
+func FindMetadataDir(bundleRoot string) (metadataDir string, err error) {
+	err = filepath.Walk(bundleRoot, func(path string, info os.FileInfo, err error) error {
+		if err != nil || !info.IsDir() {
+			return err
+		}
+		// Already found the first metadata dir, do not overwrite it.
+		if metadataDir != "" {
+			return nil
+		}
+		// The annotations file is well-defined.
+		_, err = os.Stat(filepath.Join(path, registrybundle.AnnotationsFile))
+		if err == nil || errors.Is(err, os.ErrExist) {
+			metadataDir = path
+			return nil
+		}
+		return err
+	})
+	if err != nil {
+		return "", err
+	}
+	if metadataDir == "" {
+		return "", fmt.Errorf("metadata dir not found in %s", bundleRoot)
+	}
+
+	return metadataDir, nil
+}
+
+// GetMetadataLabels reads annotations from file(s) in metadataDir and returns them as Labels.
+func GetMetadataLabels(metadataDir string) (Labels, error) {
+	// The annotations file is well-defined.
+	annotationsPath := filepath.Join(metadataDir, registrybundle.AnnotationsFile)
+	b, err := ioutil.ReadFile(annotationsPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Use the arbitrarily-labelled bundle representation of the annotations file
+	// for forwards and backwards compatibility.
+	meta := registrybundle.AnnotationMetadata{
+		Annotations: make(map[string]string),
+	}
+	if err = yaml.Unmarshal(b, &meta); err != nil {
+		return nil, err
+	}
+
+	return meta.Annotations, nil
+}

internal/registry/validate.go

@@ -27,7 +27,6 @@ import (
 	"gopkg.in/yaml.v2"
 	k8svalidation "k8s.io/apimachinery/pkg/api/validation"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"k8s.io/apimachinery/pkg/util/validation/field"
 )