test(e2e): add E2E testing utilities and sample specs

.github/workflows/build.yml

@@ -44,18 +44,21 @@ jobs:
       - name: Run E2E Tests
         run: |
           cd $GITHUB_WORKSPACE/test/e2e
-          mnt_root="$GITHUB_WORKSPACE/test/e2e/testdata/distribution/mount"
           go install github.com/onsi/ginkgo/v2/ginkgo
+          mnt_root="$GITHUB_WORKSPACE/test/e2e/testdata/distribution/mount"
           rm -rf $mnt_root/docker
           for layer in $(ls $mnt_root/*.tar.gz); do
             tar -xvzf $layer -C $mnt_root
           done
           trap 'docker kill oras-e2e || true' ERR
-          docker run -d -p 5000:5000 --rm --name oras-e2e \
-            --env STORAGE_DELETE_ENABLED=true \
+          docker run --pull always -d -p 5000:5000 --rm --name oras-e2e \
+            --env REGISTRY_STORAGE_DELETE_ENABLED=true \
+            --env REGISTRY_AUTH_HTPASSWD_REALM=test-basic \
+            --env REGISTRY_AUTH_HTPASSWD_PATH=/etc/docker/registry/passwd \
             --mount type=bind,source=$mnt_root/docker,target=/opt/data/registry-root-dir/docker \
+            --mount type=bind,source=$mnt_root/passwd_bcrypt,target=/etc/docker/registry/passwd \
             ghcr.io/oras-project/registry:v1.0.0-rc.2
-          ginkgo -r -p suite
+          ginkgo -r -p --succinct suite
           docker kill oras-e2e || true
         env:
           ORAS_PATH: bin/linux/amd64/oras

test/e2e/README.md

@@ -1,5 +1,5 @@
 # ORAS End-to-End Testing Dev Guide
-
+**KNOWN LIMITATION**: E2E tests are designed to run in the CI and currently only support running on linux platform.
 ## Setting up
 Minimal setup: Run the script in **step 3**
 
@@ -83,9 +83,9 @@ Two suites will be maintained for E2E testing:
 Inside a suite, please follow below model when building the hierarchical collections of specs:
 ```
 Describe: <Role>
-  Context: Scenario or command specific description
-    When: <Action>
-      It: <Result> (per-command execution)
+  When: Scenario or command specific description
+    It: <Action>
+      By: <Result> (per-command execution)
        Expect: <Result> (detailed checks for execution results)
 ```
 
@@ -116,4 +116,7 @@ graph TD;
     B1-- foo2 -->B2[blob1]
     B1-- bar -->B3[blob2]
     end
-```
+```
+
+#### 5.2 Scenario Suite
+Test files used by scenario-based specs are placed in `$REPO_ROOT/test/e2e/testdata/files`.

test/e2e/go.mod

@@ -5,12 +5,12 @@ go 1.19
 require (
 	github.com/onsi/ginkgo/v2 v2.1.6
 	github.com/onsi/gomega v1.20.2
+	github.com/opencontainers/go-digest v1.0.0
 	oras.land/oras-go/v2 v2.0.0-rc.3.0.20220922092058-3f9653f7bf69
 )
 
 require (
 	github.com/google/go-cmp v0.5.8 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/oras-project/artifacts-spec v1.0.0-rc.2 // indirect
 	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect

test/e2e/internal/utils/exec.go

@@ -0,0 +1,171 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	ginkgo "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gexec"
+	"oras.land/oras/test/e2e/internal/utils/match"
+)
+
+const (
+	orasBinary = "oras"
+
+	// customize your own basic auth file via `htpasswd -cBb <file_name> <user_name> <password>`
+	Username       = "hello"
+	Password       = "oras-test"
+	AuthConfigPath = "test.config"
+)
+
+// ExecOption provides option used to execute a command.
+type ExecOption struct {
+	binary  string
+	args    []string
+	workDir string
+	timeout time.Duration
+
+	stdin      io.Reader
+	stdout     []match.Matcher
+	stderr     []match.Matcher
+	shouldFail bool
+
+	text string
+}
+
+// ORAS returns default execution option for oras binary.
+func ORAS(args ...string) *ExecOption {
+	return Binary(orasBinary, args...)
+}
+
+// Binary returns default execution option for customized binary.
+func Binary(path string, args ...string) *ExecOption {
+	return &ExecOption{
+		binary:     path,
+		args:       args,
+		timeout:    10 * time.Second,
+		shouldFail: false,
+	}
+}
+
+// WithFailureCheck sets failure exit code checking for the execution.
+func (opts *ExecOption) WithFailureCheck() *ExecOption {
+	opts.shouldFail = true
+	return opts
+}
+
+// WithTimeOut sets timeout for the execution.
+func (opts *ExecOption) WithTimeOut(timeout time.Duration) *ExecOption {
+	opts.timeout = timeout
+	return opts
+}
+
+// WithDescription sets description text for the execution.
+func (opts *ExecOption) WithDescription(text string) *ExecOption {
+	opts.text = text
+	return opts
+}
+
+// WithWorkDir sets working directory for the execution.
+func (opts *ExecOption) WithWorkDir(path string) *ExecOption {
+	opts.workDir = path
+	return opts
+}
+
+// WithInput redirects stdin to r for the execution.
+func (opts *ExecOption) WithInput(r io.Reader) *ExecOption {
+	opts.stdin = r
+	return opts
+}
+
+// MatchKeyWords adds keywords matching to stdout.
+func (opts *ExecOption) MatchKeyWords(keywords ...string) *ExecOption {
+	opts.stdout = append(opts.stdout, match.NewKeywordMatcher(keywords))
+	return opts
+}
+
+// MatchErrKeyWords adds keywords matching to stderr.
+func (opts *ExecOption) MatchErrKeyWords(keywords ...string) *ExecOption {
+	opts.stderr = append(opts.stderr, match.NewKeywordMatcher(keywords))
+	return opts
+}
+
+// MatchContent adds full content matching to the execution.
+func (opts *ExecOption) MatchContent(content string) *ExecOption {
+	if !opts.shouldFail {
+		opts.stdout = append(opts.stdout, match.NewContentMatcher(content))
+	} else {
+		opts.stderr = append(opts.stderr, match.NewContentMatcher(content))
+	}
+	return opts
+}
+
+// MatchStatus adds full content matching to the execution option.
+func (opts *ExecOption) MatchStatus(keys []match.StateKey, verbose bool, successCount int) *ExecOption {
+	opts.stdout = append(opts.stdout, match.NewStatusMatcher(keys, opts.args[0], verbose, successCount))
+	return opts
+}
+
+// Exec run the execution based on opts.
+func (opts *ExecOption) Exec() *gexec.Session {
+	if opts == nil {
+		// this should be a code error but can only be caught during runtime
+		panic("Nil option for command execution")
+	}
+
+	if opts.text == "" {
+		if opts.shouldFail {
+			opts.text = "fail"
+		} else {
+			opts.text = "pass"
+		}
+	}
+	description := fmt.Sprintf(">> should %s: %s %s >>", opts.text, opts.binary, strings.Join(opts.args, " "))
+	ginkgo.By(description)
+
+	var cmd *exec.Cmd
+	if opts.binary == orasBinary {
+		opts.binary = ORASPath
+	}
+	cmd = exec.Command(opts.binary, opts.args...)
+	cmd.Stdin = opts.stdin
+	if opts.workDir != "" {
+		// switch working directory
+		wd, err := os.Getwd()
+		Expect(err).ShouldNot(HaveOccurred())
+		Expect(os.Chdir(opts.workDir)).ShouldNot(HaveOccurred())
+		defer os.Chdir(wd)
+	}
+	fmt.Println(description)
+	session, err := gexec.Start(cmd, os.Stdout, os.Stderr)
+	Expect(err).ShouldNot(HaveOccurred())
+	Expect(session.Wait(opts.timeout).ExitCode() != 0).Should(Equal(opts.shouldFail))
+
+	// matching result
+	for _, s := range opts.stdout {
+		s.Match(session.Out)
+	}
+	for _, s := range opts.stderr {
+		s.Match(session.Err)
+	}
+
+	return session
+}

test/e2e/internal/utils/file.go

@@ -0,0 +1,66 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+var testFileRoot string
+
+// CopyTestData copies test data into the temp test folder.
+func CopyTestData(dstRoot string) error {
+	return filepath.WalkDir(testFileRoot, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			// ignore folder
+			return nil
+		}
+
+		relPath, err := filepath.Rel(testFileRoot, path)
+		if err != nil {
+			return err
+		}
+		dstPath := filepath.Join(dstRoot, relPath)
+		// make sure all parents are created
+		if err := os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil {
+			return err
+		}
+
+		// copy with original folder structure
+		return copyFile(path, dstPath)
+	})
+}
+
+func copyFile(srcFile, dstFile string) error {
+	to, err := os.Create(dstFile)
+	if err != nil {
+		return err
+	}
+	defer to.Close()
+
+	from, err := os.Open(srcFile)
+	if err != nil {
+		return err
+	}
+	defer from.Close()
+
+	_, err = io.Copy(to, from)
+	return err
+}

test/e2e/internal/utils/init.go

@@ -16,6 +16,7 @@ package utils
 import (
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -42,27 +43,31 @@ func init() {
 	if err := ref.ValidateRegistry(); err != nil {
 		panic(err)
 	}
+	// setup test data
+	pwd, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+	testFileRoot = filepath.Join(pwd, "..", "..", "testdata", "files")
 	BeforeSuite(func() {
 		ORASPath = os.Getenv("ORAS_PATH")
 		if filepath.IsAbs(ORASPath) {
 			fmt.Printf("Testing based on pre-built binary locates in %q\n", ORASPath)
-			return
-		}
-
-		var err error
-		if workspacePath := os.Getenv("GITHUB_WORKSPACE"); ORASPath != "" && workspacePath != "" {
+		} else if workspacePath := os.Getenv("GITHUB_WORKSPACE"); ORASPath != "" && workspacePath != "" {
 			// add workspacePath as prefix, both path env should not be empty
 			ORASPath = filepath.Join(workspacePath, ORASPath)
 			ORASPath, err = filepath.Abs(ORASPath)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 			fmt.Printf("Testing based on pre-built binary locates in %q\n", ORASPath)
-			return
+		} else {
+			// fallback to native build to facilitate local debugging
+			ORASPath, err = gexec.Build("oras.land/oras/cmd/oras")
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+			DeferCleanup(gexec.CleanupBuildArtifacts)
+			fmt.Printf("Testing based on temp binary locates in %q\n", ORASPath)
 		}
 
-		// fallback to native build to facilitate local debugging
-		ORASPath, err = gexec.Build("oras.land/oras/cmd/oras")
-		gomega.Expect(err).NotTo(gomega.HaveOccurred())
-		DeferCleanup(gexec.CleanupBuildArtifacts)
-		fmt.Printf("Testing based on temp binary locates in %q\n", ORASPath)
+		cmd := exec.Command(ORASPath, "login", Host, "-u", Username, "-p", Password)
+		gomega.Expect(cmd.Run()).ShouldNot(gomega.HaveOccurred())
 	})
 }

test/e2e/internal/utils/match/content.go

@@ -0,0 +1,33 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package match
+
+import (
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gbytes"
+)
+
+// contentMatcher provides whole matching of the output.
+type contentMatcher string
+
+// NewContentMatcher returns a content matcher.
+func NewContentMatcher(s string) contentMatcher {
+	return contentMatcher(s)
+}
+
+// Match matches got with s.
+func (c contentMatcher) Match(got *gbytes.Buffer) {
+	content := got.Contents()
+	Expect(contentMatcher(content)).Should(Equal(c))
+}