-
Notifications
You must be signed in to change notification settings - Fork 164
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
test(e2e): add E2E testing utilities and sample specs (#639)
This PR includes below changes in the E2E utility package: 1. Enables basic auth in test infra 2. Supports excuting oras or other binaries with fluent-styled runtime configuring and result matching 3. Introduces `match` packages to validate execution results 4. Added sample test specs for `manifest fetch` command and common OCI image scenario. Resolves #554, resolves #555, resolves #556, resolves #565 Signed-off-by: Billy Zha <jinzha1@microsoft.com>
- Loading branch information
Showing
22 changed files
with
957 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,171 @@ | ||
| /* | ||
| Copyright The ORAS Authors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package utils | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "os/exec" | ||
| "strings" | ||
| "time" | ||
|
|
||
| ginkgo "github.com/onsi/ginkgo/v2" | ||
| . "github.com/onsi/gomega" | ||
| "github.com/onsi/gomega/gexec" | ||
| "oras.land/oras/test/e2e/internal/utils/match" | ||
| ) | ||
|
|
||
| const ( | ||
| orasBinary = "oras" | ||
|
|
||
| // customize your own basic auth file via `htpasswd -cBb <file_name> <user_name> <password>` | ||
| Username = "hello" | ||
| Password = "oras-test" | ||
| AuthConfigPath = "test.config" | ||
| ) | ||
|
|
||
| // ExecOption provides option used to execute a command. | ||
| type ExecOption struct { | ||
| binary string | ||
| args []string | ||
| workDir string | ||
| timeout time.Duration | ||
|
|
||
| stdin io.Reader | ||
| stdout []match.Matcher | ||
| stderr []match.Matcher | ||
| shouldFail bool | ||
|
|
||
| text string | ||
| } | ||
|
|
||
| // ORAS returns default execution option for oras binary. | ||
| func ORAS(args ...string) *ExecOption { | ||
| return Binary(orasBinary, args...) | ||
| } | ||
|
|
||
| // Binary returns default execution option for customized binary. | ||
| func Binary(path string, args ...string) *ExecOption { | ||
| return &ExecOption{ | ||
| binary: path, | ||
| args: args, | ||
| timeout: 10 * time.Second, | ||
| shouldFail: false, | ||
| } | ||
| } | ||
|
|
||
| // WithFailureCheck sets failure exit code checking for the execution. | ||
| func (opts *ExecOption) WithFailureCheck() *ExecOption { | ||
| opts.shouldFail = true | ||
| return opts | ||
| } | ||
|
|
||
| // WithTimeOut sets timeout for the execution. | ||
| func (opts *ExecOption) WithTimeOut(timeout time.Duration) *ExecOption { | ||
| opts.timeout = timeout | ||
| return opts | ||
| } | ||
|
|
||
| // WithDescription sets description text for the execution. | ||
| func (opts *ExecOption) WithDescription(text string) *ExecOption { | ||
| opts.text = text | ||
| return opts | ||
| } | ||
|
|
||
| // WithWorkDir sets working directory for the execution. | ||
| func (opts *ExecOption) WithWorkDir(path string) *ExecOption { | ||
| opts.workDir = path | ||
| return opts | ||
| } | ||
|
|
||
| // WithInput redirects stdin to r for the execution. | ||
| func (opts *ExecOption) WithInput(r io.Reader) *ExecOption { | ||
| opts.stdin = r | ||
| return opts | ||
| } | ||
|
|
||
| // MatchKeyWords adds keywords matching to stdout. | ||
| func (opts *ExecOption) MatchKeyWords(keywords ...string) *ExecOption { | ||
| opts.stdout = append(opts.stdout, match.NewKeywordMatcher(keywords)) | ||
| return opts | ||
| } | ||
|
|
||
| // MatchErrKeyWords adds keywords matching to stderr. | ||
| func (opts *ExecOption) MatchErrKeyWords(keywords ...string) *ExecOption { | ||
| opts.stderr = append(opts.stderr, match.NewKeywordMatcher(keywords)) | ||
| return opts | ||
| } | ||
|
|
||
| // MatchContent adds full content matching to the execution. | ||
| func (opts *ExecOption) MatchContent(content string) *ExecOption { | ||
| if !opts.shouldFail { | ||
| opts.stdout = append(opts.stdout, match.NewContentMatcher(content)) | ||
| } else { | ||
| opts.stderr = append(opts.stderr, match.NewContentMatcher(content)) | ||
| } | ||
| return opts | ||
| } | ||
|
|
||
| // MatchStatus adds full content matching to the execution option. | ||
| func (opts *ExecOption) MatchStatus(keys []match.StateKey, verbose bool, successCount int) *ExecOption { | ||
| opts.stdout = append(opts.stdout, match.NewStatusMatcher(keys, opts.args[0], verbose, successCount)) | ||
| return opts | ||
| } | ||
|
|
||
| // Exec run the execution based on opts. | ||
| func (opts *ExecOption) Exec() *gexec.Session { | ||
| if opts == nil { | ||
| // this should be a code error but can only be caught during runtime | ||
| panic("Nil option for command execution") | ||
| } | ||
|
|
||
| if opts.text == "" { | ||
| if opts.shouldFail { | ||
| opts.text = "fail" | ||
| } else { | ||
| opts.text = "pass" | ||
| } | ||
| } | ||
| description := fmt.Sprintf(">> should %s: %s %s >>", opts.text, opts.binary, strings.Join(opts.args, " ")) | ||
| ginkgo.By(description) | ||
|
|
||
| var cmd *exec.Cmd | ||
| if opts.binary == orasBinary { | ||
| opts.binary = ORASPath | ||
| } | ||
| cmd = exec.Command(opts.binary, opts.args...) | ||
| cmd.Stdin = opts.stdin | ||
| if opts.workDir != "" { | ||
| // switch working directory | ||
| wd, err := os.Getwd() | ||
| Expect(err).ShouldNot(HaveOccurred()) | ||
| Expect(os.Chdir(opts.workDir)).ShouldNot(HaveOccurred()) | ||
| defer os.Chdir(wd) | ||
| } | ||
| fmt.Println(description) | ||
| session, err := gexec.Start(cmd, os.Stdout, os.Stderr) | ||
| Expect(err).ShouldNot(HaveOccurred()) | ||
| Expect(session.Wait(opts.timeout).ExitCode() != 0).Should(Equal(opts.shouldFail)) | ||
|
|
||
| // matching result | ||
| for _, s := range opts.stdout { | ||
| s.Match(session.Out) | ||
| } | ||
| for _, s := range opts.stderr { | ||
| s.Match(session.Err) | ||
| } | ||
|
|
||
| return session | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| /* | ||
| Copyright The ORAS Authors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package utils | ||
|
|
||
| import ( | ||
| "io" | ||
| "io/fs" | ||
| "os" | ||
| "path/filepath" | ||
| ) | ||
|
|
||
| var testFileRoot string | ||
|
|
||
| // CopyTestData copies test data into the temp test folder. | ||
| func CopyTestData(dstRoot string) error { | ||
| return filepath.WalkDir(testFileRoot, func(path string, d fs.DirEntry, err error) error { | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if d.IsDir() { | ||
| // ignore folder | ||
| return nil | ||
| } | ||
|
|
||
| relPath, err := filepath.Rel(testFileRoot, path) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| dstPath := filepath.Join(dstRoot, relPath) | ||
| // make sure all parents are created | ||
| if err := os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // copy with original folder structure | ||
| return copyFile(path, dstPath) | ||
| }) | ||
| } | ||
|
|
||
| func copyFile(srcFile, dstFile string) error { | ||
| to, err := os.Create(dstFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer to.Close() | ||
|
|
||
| from, err := os.Open(srcFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer from.Close() | ||
|
|
||
| _, err = io.Copy(to, from) | ||
| return err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| /* | ||
| Copyright The ORAS Authors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package match | ||
|
|
||
| import ( | ||
| . "github.com/onsi/gomega" | ||
| "github.com/onsi/gomega/gbytes" | ||
| ) | ||
|
|
||
| // contentMatcher provides whole matching of the output. | ||
| type contentMatcher string | ||
|
|
||
| // NewContentMatcher returns a content matcher. | ||
| func NewContentMatcher(s string) contentMatcher { | ||
| return contentMatcher(s) | ||
| } | ||
|
|
||
| // Match matches got with s. | ||
| func (c contentMatcher) Match(got *gbytes.Buffer) { | ||
| content := got.Contents() | ||
| Expect(contentMatcher(content)).Should(Equal(c)) | ||
| } |
Oops, something went wrong.