Scorecard2 pod execution

.gitignore

@@ -97,6 +97,7 @@ tags
 test/ansible-operator/ansible-operator
 test/helm-operator/helm-operator
 images/scorecard-proxy/scorecard-proxy
+images/scorecard-test/scorecard-test
 
 # Test artifacts
 pkg/ansible/runner/testdata/valid.yaml

cmd/operator-sdk/alpha/scorecard/cmd.go

@@ -16,7 +16,6 @@ package scorecard
 
 import (
 	"fmt"
-	"log"
 
 	scorecard "github.com/operator-framework/operator-sdk/internal/scorecard/alpha"
 	"github.com/spf13/cobra"
@@ -25,10 +24,16 @@ import (
 
 func NewCmd() *cobra.Command {
 	var (
-		config string
-		//bundle   string // TODO - to be implemented
-		selector string
-		//list     bool // TODO - to be implemented
+		config         string
+		output         string
+		bundle         string
+		selector       string
+		kubeconfig     string
+		namespace      string
+		serviceAccount string
+		list           bool
+		cleanup        bool
+		waitTime       int
 	)
 	scorecardCmd := &cobra.Command{
 		Use:    "scorecard",
@@ -38,31 +43,53 @@ func NewCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 
 			var err error
-			o := scorecard.Options{}
+			o := scorecard.Options{
+				ServiceAccount: serviceAccount,
+				Namespace:      namespace,
+				List:           list,
+				BundlePath:     bundle,
+				OutputFormat:   output,
+				Cleanup:        cleanup,
+				WaitTime:       waitTime,
+			}
+			o.Client, err = scorecard.GetKubeClient(kubeconfig)
+			if err != nil {
+				return fmt.Errorf("could not get Kube connection %s", err.Error())
+			}
 			o.Config, err = scorecard.LoadConfig(config)
 			if err != nil {
 				return fmt.Errorf("could not find config file %s", err.Error())
 			}
 
+			if bundle == "" {
+				return fmt.Errorf("bundle flag required")
+			}
+
 			o.Selector, err = labels.Parse(selector)
 			if err != nil {
 				return fmt.Errorf("could not parse selector %s", err.Error())
 			}
 
-			// TODO - process list and output formatting here?
-
-			if err := scorecard.RunTests(o); err != nil {
-				log.Fatal(err)
+			if list {
+				return scorecard.ListTests(o)
 			}
-			return nil
+
+			return scorecard.RunTests(o)
 		},
 	}
 
 	scorecardCmd.Flags().StringVarP(&config, "config", "c", "",
 		"path to a new to be defined DSL yaml formatted file that configures what tests get executed")
-	//	scorecardCmd.Flags().StringVar(&bundle, "bundle", "", "path to the operator bundle contents on disk")
+	scorecardCmd.Flags().StringVar(&kubeconfig, "kubeconfig", "", "kubeconfig path")
+
+	scorecardCmd.Flags().StringVar(&bundle, "bundle", "", "path to the operator bundle contents on disk")
 	scorecardCmd.Flags().StringVarP(&selector, "selector", "l", "", "label selector to determine which tests are run")
-	//	scorecardCmd.Flags().BoolVarP(&list, "list", "L", false, "option to enable listing which tests are run")
+	scorecardCmd.Flags().StringVarP(&namespace, "namespace", "n", "default", "namespace to run the test images in")
+	scorecardCmd.Flags().StringVarP(&output, "output", "o", "text", "Output format for results.  Valid values: text, json")
+	scorecardCmd.Flags().StringVarP(&serviceAccount, "service-account", "s", "default", "service account to use for tests")
+	scorecardCmd.Flags().BoolVarP(&list, "list", "L", false, "option to enable listing which tests are run")
+	scorecardCmd.Flags().BoolVarP(&cleanup, "cleanup", "x", true, "option to disable resource cleanup after tests are run")
+	scorecardCmd.Flags().IntVarP(&waitTime, "wait-time", "w", 30, "time in seconds to wait for tests to complete")
 
 	return scorecardCmd
 }

hack/image/build-scorecard-test-image.sh

@@ -4,14 +4,14 @@ set -eux
 
 source hack/lib/image_lib.sh
 
-# TODO build test image
-#WD="$(dirname "$(pwd)")"
-#GOOS=linux CGO_ENABLED=0 \
-#  go build \
-#  -gcflags "all=-trimpath=${WD}" \
-#  -asmflags "all=-trimpath=${WD}" \
-#  -o images/scorecard-test/scorecard-test \
-#  images/scorecard-test/cmd/test/main.go
+# build scorecard test image
+WD="$(dirname "$(pwd)")"
+GOOS=linux CGO_ENABLED=0 \
+  go build \
+  -gcflags "all=-trimpath=${WD}" \
+  -asmflags "all=-trimpath=${WD}" \
+  -o images/scorecard-test/scorecard-test \
+  images/scorecard-test/cmd/test/main.go
 
 # Build base image
 pushd images/scorecard-test

images/scorecard-test/Dockerfile

@@ -5,13 +5,12 @@ ENV TEST=/usr/local/bin/scorecard-test \
     USER_UID=1001 \
     USER_NAME=test
 
-# TODO install test binary 
-# COPY scorecard-test ${TEST}
+# install test binary 
+COPY scorecard-test ${TEST}
 
 COPY bin /usr/local/bin
 RUN  /usr/local/bin/user_setup
 
-
 ENTRYPOINT ["/usr/local/bin/entrypoint"]
 
 USER ${USER_UID}

images/scorecard-test/cmd/test/main.go

@@ -17,11 +17,14 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 
+	"github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha2"
 	scapiv1alpha2 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha2"
 
+	"github.com/operator-framework/operator-sdk/internal/scorecard/alpha"
 	"github.com/operator-framework/operator-sdk/internal/scorecard/alpha/tests"
 )
 
@@ -35,7 +38,8 @@ import (
 // test image.
 
 const (
-	bundlePath = "/scorecard"
+	// bundleTar is the tar file containing the bundle contents
+	bundleTar = "/scorecard/bundle.tar"
 )
 
 func main() {
@@ -44,7 +48,19 @@ func main() {
 		log.Fatal("test name argument is required")
 	}
 
-	cfg, err := tests.GetBundle(bundlePath)
+	// Create tmp directory for the untar'd bundle
+	tmpDir, err := ioutil.TempDir("/tmp", "scorecard-bundle")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.Remove(tmpDir)
+
+	err = alpha.Untartar(bundleTar, tmpDir)
+	if err != nil {
+		log.Fatalf("error untarring bundle %s", err.Error())
+	}
+
+	cfg, err := tests.GetBundle(tmpDir)
 	if err != nil {
 		log.Fatal(err.Error())
 	}
@@ -62,14 +78,10 @@ func main() {
 		result = tests.SpecDescriptorsTest(cfg)
 	case tests.OLMStatusDescriptorsTest:
 		result = tests.StatusDescriptorsTest(cfg)
-	case tests.BasicCheckStatusTest:
-		result = tests.CheckStatusTest(cfg)
 	case tests.BasicCheckSpecTest:
 		result = tests.CheckSpecTest(cfg)
 	default:
-		log.Fatal("invalid test name argument passed")
-		// TODO print out full list of test names to give a hint
-		// to the end user on what the valid tests are
+		result = printValidTests()
 	}
 
 	prettyJSON, err := json.MarshalIndent(result, "", "    ")
@@ -79,3 +91,20 @@ func main() {
 	fmt.Printf("%s\n", string(prettyJSON))
 
 }
+
+// printValidTests will print out full list of test names to give a hint to the end user on what the valid tests are
+func printValidTests() (result v1alpha2.ScorecardTestResult) {
+	result.State = scapiv1alpha2.FailState
+	result.Errors = make([]string, 0)
+	result.Suggestions = make([]string, 0)
+
+	str := fmt.Sprintf("Valid tests for this image include: %s, %s, %s, %s, %s, %s",
+		tests.OLMBundleValidationTest,
+		tests.OLMCRDsHaveValidationTest,
+		tests.OLMCRDsHaveResourcesTest,
+		tests.OLMSpecDescriptorsTest,
+		tests.OLMStatusDescriptorsTest,
+		tests.BasicCheckSpecTest)
+	result.Suggestions = append(result.Suggestions, str)
+	return result
+}

internal/scorecard/alpha/bundle.go

@@ -0,0 +1,49 @@
+// Copyright 2020 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package alpha
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+)
+
+// getBundleData tars up the contents of a bundle from a path, and returns that tar file in []byte
+func getBundleData(bundlePath string) (bundleData []byte, err error) {
+
+	// make sure the bundle exists on disk
+	_, err = os.Stat(bundlePath)
+	if os.IsNotExist(err) {
+		return bundleData, fmt.Errorf("bundle path is not valid %s", err.Error())
+	}
+
+	tempTarFileName := fmt.Sprintf("%s%ctempBundle-%s.tar", os.TempDir(), os.PathSeparator, randomString())
+
+	paths := []string{bundlePath}
+	err = Tartar(tempTarFileName, paths)
+	if err != nil {
+		return bundleData, fmt.Errorf("error creating tar of bundle %s", err.Error())
+	}
+
+	defer os.Remove(tempTarFileName)
+
+	var buf []byte
+	buf, err = ioutil.ReadFile(tempTarFileName)
+	if err != nil {
+		return bundleData, fmt.Errorf("error reading tar of bundle %s", err.Error())
+	}
+
+	return buf, err
+}

internal/scorecard/alpha/config.go

@@ -14,16 +14,43 @@
 
 package alpha
 
+import (
+	"io/ioutil"
+
+	"gopkg.in/yaml.v2"
+	v1 "k8s.io/api/core/v1"
+)
+
 type ScorecardTest struct {
 	Name        string            `yaml:"name"`                 // The container test name
 	Image       string            `yaml:"image"`                // The container image name
 	Entrypoint  string            `yaml:"entrypoint,omitempty"` // An optional entrypoint passed to the test image
 	Labels      map[string]string `yaml:"labels"`               // User defined labels used to filter tests
 	Description string            `yaml:"description"`          // User readable test description
+	TestPod     *v1.Pod           `yaml:"-"`                    // Pod that ran the test
 }
 
 // Config represents the set of test configurations which scorecard
 // would run based on user input
 type Config struct {
 	Tests []ScorecardTest `yaml:"tests"`
 }
+
+// LoadConfig will find and return the scorecard config, the config file
+// can be passed in via command line flag or from a bundle location or
+// bundle image
+func LoadConfig(configFilePath string) (Config, error) {
+	c := Config{}
+
+	// TODO handle getting config from bundle (ondisk or image)
+	yamlFile, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return c, err
+	}
+
+	if err := yaml.Unmarshal(yamlFile, &c); err != nil {
+		return c, err
+	}
+
+	return c, nil
+}

internal/scorecard/alpha/config_test.go

@@ -30,11 +30,12 @@ func TestInvalidConfigPath(t *testing.T) {
 	for _, c := range cases {
 		t.Run(c.configPathValue, func(t *testing.T) {
 			_, err := LoadConfig(c.configPathValue)
-			if err != nil && c.wantError {
-				t.Logf("Wanted error and got error : %v", err)
-				return
-			} else if err != nil && !c.wantError {
-				t.Errorf("Wanted result but got error: %v", err)
+			if err == nil && c.wantError {
+				t.Fatalf("Wanted error but got no error")
+			} else if err != nil {
+				if !c.wantError {
+					t.Fatalf("Wanted result but got error: %v", err)
+				}
 				return
 			}