From e929f2874ce4f38c56fccc1651cef223eb6f7467 Mon Sep 17 00:00:00 2001 From: David Timm Date: Tue, 9 May 2023 12:29:07 -0600 Subject: [PATCH] feat: add migrations test workflow to test command - default `kiln test` runs migration and manifest tests now - flags `--manifest-only` and `--migrations-only` added to test command - Node.js/npm added to Dockerfile for test command Co-authored-by: Preethi Varambally Co-authored-by: Pablo Rodas --- .../commands/manifest_test_docker/Dockerfile | 9 ++ internal/commands/test_tile.go | 34 +++++-- .../commands/test_tile_acceptance_test.go | 9 +- internal/commands/test_tile_test.go | 96 +++++++++++++++++-- .../tas_fake/tas/migrations/.gitignore | 2 + .../this-file-should-be-excluded.js | 0 .../tas_fake/tas/migrations/package.json | 15 +++ main.go | 2 +- 8 files changed, 145 insertions(+), 22 deletions(-) create mode 100644 internal/commands/testdata/tas_fake/tas/migrations/.gitignore delete mode 100644 internal/commands/testdata/tas_fake/tas/migrations/node_modules/this-file-should-be-excluded.js create mode 100644 internal/commands/testdata/tas_fake/tas/migrations/package.json diff --git a/internal/commands/manifest_test_docker/Dockerfile b/internal/commands/manifest_test_docker/Dockerfile index 1fe426f8c..91c0b491b 100644 --- a/internal/commands/manifest_test_docker/Dockerfile +++ b/internal/commands/manifest_test_docker/Dockerfile @@ -23,6 +23,15 @@ RUN apt-get update && apt-get install jq -y # Install Ginkgo RUN go install github.com/onsi/ginkgo/ginkgo@latest && touch $HOME/.ack-ginkgo-rc +# Install NPM +ENV NODE_VERSION=18.16.0 +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash +ENV NVM_DIR=/root/.nvm +RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION} +ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin:${PATH}" + # Install ops-manifest # assumes ops-manifest repo was cloned into ./vendor/ops-manager/gems/ops-manifest COPY --from=builder /tmp/ops-manager /tmp/ops-manager diff --git a/internal/commands/test_tile.go b/internal/commands/test_tile.go index 11fe7973c..ab4829a69 100644 --- a/internal/commands/test_tile.go +++ b/internal/commands/test_tile.go @@ -72,11 +72,13 @@ func (l infoLog) Writer() io.Writer { return l.logger.Writer() } -type ManifestTest struct { +type TileTest struct { Options struct { TilePath string `short:"tp" long:"tile-path" default:"." description:"path to the tile directory (e.g., ~/workspace/tas/ist)"` GingkoManifestFlags string `short:"gmf" long:"ginkgo-manifest-flags" default:"-r -p -slowSpecThreshold 15" description:"flags to pass to the gingko manifest test suite"` Verbose bool `short:"v" long:"verbose" default:"false" description:"log info lines. this doesn't apply to ginkgo.'"` + ManifestOnly bool ` long:"manifest-only" default:"false" description:"run only manifest tests"` + MigrationsOnly bool ` long:"migrations-only" default:"false" description:"run only migration tests"` } logger *log.Logger @@ -86,9 +88,9 @@ type ManifestTest struct { sshProvider SshProvider } -func NewManifestTest(logger *log.Logger, ctx context.Context, mobi mobyClient, sshThing SshProvider) ManifestTest { +func NewTileTest(logger *log.Logger, ctx context.Context, mobi mobyClient, sshThing SshProvider) TileTest { ctx, cancelFunc := context.WithCancel(ctx) - return ManifestTest{ + return TileTest{ ctx: ctx, cancelFunc: cancelFunc, logger: logger, @@ -101,7 +103,7 @@ func NewManifestTest(logger *log.Logger, ctx context.Context, mobi mobyClient, s //go:embed manifest_test_docker/* var dockerfileContents string -func (u ManifestTest) Execute(args []string) error { +func (u TileTest) Execute(args []string) error { if u.sshProvider == nil { return errors.New("ssh provider failed to initialize. check your ssh-agent is running") } @@ -191,8 +193,22 @@ func (u ManifestTest) Execute(args []string) error { loggerWithInfo.Info("Mounting", parentDir, "and testing", tileDir) - envVars := getManifestTestEnvVars(absRepoDir, tileDir) - dockerCmd := fmt.Sprintf("cd /tas/%s/test/manifest && PRODUCT=%s RENDERER=ops-manifest ginkgo %s", tileDir, toProduct(tileDir), u.Options.GingkoManifestFlags) + runAll := !u.Options.ManifestOnly && !u.Options.MigrationsOnly + + var dockerCmds []string + if u.Options.ManifestOnly || runAll { + dockerCmds = append(dockerCmds, fmt.Sprintf("cd /tas/%s/test/manifest", tileDir)) + dockerCmds = append(dockerCmds, fmt.Sprintf("PRODUCT=%s RENDERER=ops-manifest ginkgo %s", toProduct(tileDir), u.Options.GingkoManifestFlags)) + } + if u.Options.MigrationsOnly || runAll { + dockerCmds = append(dockerCmds, fmt.Sprintf("cd /tas/%s/migrations", tileDir)) + dockerCmds = append(dockerCmds, "npm install") + dockerCmds = append(dockerCmds, "npm test") + } + + dockerCmd := strings.Join(dockerCmds, " && ") + + envVars := getTileTestEnvVars(absRepoDir, tileDir) loggerWithInfo.Info("Running:", dockerCmd) createResp, err := u.mobi.ContainerCreate(u.ctx, &container.Config{ Image: "kiln_test_dependencies:vmware", @@ -272,7 +288,7 @@ func toProduct(dir string) string { } } -func getManifestTestEnvVars(dir, productDir string) []string { +func getTileTestEnvVars(dir, productDir string) []string { const fixturesFormat = "%s/test/manifest/fixtures" metadataPath := fmt.Sprintf(fixturesFormat+"/tas_metadata.yml", dir) configPath := fmt.Sprintf(fixturesFormat+"/tas_config.yml", dir) @@ -313,7 +329,7 @@ func getTarReader(fileContents string) (*bufio.Reader, error) { return tr, nil } -func (u ManifestTest) addMissingKeys() error { +func (u TileTest) addMissingKeys() error { needsKeys, err := u.sshProvider.NeedsKeys() if needsKeys { key, err := u.sshProvider.GetKeys() @@ -354,7 +370,7 @@ type ErrorLine struct { Error string `json:"error"` } -func (u ManifestTest) Usage() jhanda.Usage { +func (u TileTest) Usage() jhanda.Usage { return jhanda.Usage{ Description: "Test the manifest for a product inside a docker container. Requires a docker daemon to be running and ssh keys with access to Ops Manager's git repo. For non-interactive use either set the env var SSH_PASSWORD or add your ssh identify before running.", ShortDescription: "Test manifest for a product", diff --git a/internal/commands/test_tile_acceptance_test.go b/internal/commands/test_tile_acceptance_test.go index 11d94ec3a..0a9dd7bbc 100644 --- a/internal/commands/test_tile_acceptance_test.go +++ b/internal/commands/test_tile_acceptance_test.go @@ -16,7 +16,7 @@ import ( ) var _ = Describe("test", func() { - Context("manifest tests succeed", func() { + Context("all tests succeed", func() { It("succeeds", func() { var testOutput bytes.Buffer logger := log.New(&testOutput, "", 0) @@ -28,16 +28,17 @@ var _ = Describe("test", func() { Expect(err).NotTo(HaveOccurred()) tilePath := filepath.Join("testdata", "tas_fake", "tas") Expect(goVendor(tilePath)).NotTo(HaveOccurred()) - testTile := commands.NewManifestTest(logger, ctx, cli, sshProvider) + testTile := commands.NewTileTest(logger, ctx, cli, sshProvider) err = testTile.Execute([]string{"--verbose", "--tile-path", tilePath}) Expect(err).NotTo(HaveOccurred()) Expect(testOutput.String()).To(ContainSubstring("SUCCESS")) + Expect(testOutput.String()).To(ContainSubstring("hello, world")) Expect(testOutput.String()).NotTo(ContainSubstring("Failure")) }) }) - Context("manifest tests fail", func() { + Context("all tests fail", func() { It("fails", func() { var testOutput bytes.Buffer logger := log.New(&testOutput, "", 0) @@ -47,7 +48,7 @@ var _ = Describe("test", func() { sshProvider, err := commands.NewSshProvider(commands.SSHClientCreator{}) Expect(err).NotTo(HaveOccurred()) - testTile := commands.NewManifestTest(logger, ctx, cli, sshProvider) + testTile := commands.NewTileTest(logger, ctx, cli, sshProvider) tilePath := filepath.Join("testdata", "tas_fake", "tas_failing") Expect(goVendor(tilePath)).NotTo(HaveOccurred()) err = testTile.Execute([]string{"--verbose", "--tile-path", tilePath}) diff --git a/internal/commands/test_tile_test.go b/internal/commands/test_tile_test.go index 9f17a684a..11806eb91 100644 --- a/internal/commands/test_tile_test.go +++ b/internal/commands/test_tile_test.go @@ -47,7 +47,7 @@ var _ = Describe("kiln test docker", func() { logger = log.New(&writer, "", 0) }) - Describe("successful creation creation", func() { + Describe("test outcomes", func() { var ( fakeSshProvider *fakes.SshProvider helloTilePath string @@ -73,15 +73,15 @@ var _ = Describe("kiln test docker", func() { }) When("executing tests", func() { var ( - subjectUnderTest commands.ManifestTest + subjectUnderTest commands.TileTest ) BeforeEach(func() { writer.Reset() - subjectUnderTest = commands.NewManifestTest(logger, ctx, fakeMobyClient, fakeSshProvider) + subjectUnderTest = commands.NewTileTest(logger, ctx, fakeMobyClient, fakeSshProvider) }) When("verbose is passed", func() { It("succeeds and logs info", func() { - err := subjectUnderTest.Execute([]string{"--verbose", "--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"}) + err := subjectUnderTest.Execute([]string{"--manifest-only", "--verbose", "--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"}) Expect(err).To(BeNil()) By("logging helpful messages", func() { @@ -112,7 +112,7 @@ var _ = Describe("kiln test docker", func() { }) When("verbose isn't passed", func() { It("doesn't log info", func() { - err := subjectUnderTest.Execute([]string{"--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"}) + err := subjectUnderTest.Execute([]string{"--manifest-only", "--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"}) Expect(err).To(BeNil()) By("logging helpful messages", func() { @@ -141,8 +141,8 @@ var _ = Describe("kiln test docker", func() { fakeMobyClient = setupFakeMobyClient(testFailureMessage, 1) }) It("returns an error", func() { - subjectUnderTest := commands.NewManifestTest(logger, ctx, fakeMobyClient, fakeSshProvider) - err := subjectUnderTest.Execute([]string{"--verbose", "--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"}) + subjectUnderTest := commands.NewTileTest(logger, ctx, fakeMobyClient, fakeSshProvider) + err := subjectUnderTest.Execute([]string{"--manifest-only", "--verbose", "--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"}) Expect(err).To(HaveOccurred()) By("logging helpful messages", func() { @@ -153,6 +153,86 @@ var _ = Describe("kiln test docker", func() { }) }) }) + + When("all tests are run", func() { + var ( + fakeMobyClient *fakes.MobyClient + ) + BeforeEach(func() { + fakeMobyClient = setupFakeMobyClient("success", 0) + }) + When("executing migration tests", func() { + var ( + subjectUnderTest commands.TileTest + ) + BeforeEach(func() { + writer.Reset() + subjectUnderTest = commands.NewTileTest(logger, ctx, fakeMobyClient, fakeSshProvider) + }) + + It("succeeds", func() { + err := subjectUnderTest.Execute([]string{"--tile-path", helloTilePath}) + Expect(err).To(BeNil()) + + By("creating a test container", func() { + Expect(fakeMobyClient.ContainerCreateCallCount()).To(Equal(1)) + _, config, _, _, _, _ := fakeMobyClient.ContainerCreateArgsForCall(0) + + By("executing the tests", func() { + dockerCmd := "cd /tas/hello-tile/test/manifest && PRODUCT=hello-tile RENDERER=ops-manifest ginkgo -r -p -slowSpecThreshold 15 && cd /tas/hello-tile/migrations && npm install && npm test" + Expect(config.Cmd).To(Equal(strslice.StrSlice{"/bin/bash", "-c", dockerCmd})) + }) + }) + }) + }) + }) + + When("migration tests should be successful", func() { + const ( + testSuccessLogLine = "migration tests completed successfully" + ) + var ( + fakeMobyClient *fakes.MobyClient + ) + BeforeEach(func() { + fakeMobyClient = setupFakeMobyClient(testSuccessLogLine, 0) + }) + When("executing migration tests", func() { + var ( + subjectUnderTest commands.TileTest + ) + BeforeEach(func() { + writer.Reset() + subjectUnderTest = commands.NewTileTest(logger, ctx, fakeMobyClient, fakeSshProvider) + }) + + It("succeeds and logs info", func() { + err := subjectUnderTest.Execute([]string{"--migrations-only", "--verbose", "--tile-path", helloTilePath}) + Expect(err).To(BeNil()) + + By("logging helpful messages", func() { + logs := writer.String() + By("logging container information", func() { + Expect(logs).To(ContainSubstring("Building / restoring cached docker image")) + }) + By("logging test lines", func() { + Expect(logs).To(ContainSubstring("migration tests completed successfully")) + }) + }) + + By("creating a test container", func() { + Expect(fakeMobyClient.ContainerCreateCallCount()).To(Equal(1)) + _, config, _, _, _, _ := fakeMobyClient.ContainerCreateArgsForCall(0) + + By("executing the tests", func() { + dockerCmd := "cd /tas/hello-tile/migrations && npm install && npm test" + Expect(config.Cmd).To(Equal(strslice.StrSlice{"/bin/bash", "-c", dockerCmd})) + }) + }) + }) + }) + }) + }) It("exits with an error if docker isn't running", func() { @@ -160,7 +240,7 @@ var _ = Describe("kiln test docker", func() { fakeMobyClient.PingReturns(types.Ping{}, errors.New("docker not running")) fakeSshThinger := fakes.SshProvider{} fakeSshThinger.NeedsKeysReturns(false, nil) - subjectUnderTest := commands.NewManifestTest(logger, ctx, fakeMobyClient, &fakeSshThinger) + subjectUnderTest := commands.NewTileTest(logger, ctx, fakeMobyClient, &fakeSshThinger) err := subjectUnderTest.Execute([]string{filepath.Join(helloTileDirectorySegments...)}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("Docker daemon is not running")) diff --git a/internal/commands/testdata/tas_fake/tas/migrations/.gitignore b/internal/commands/testdata/tas_fake/tas/migrations/.gitignore new file mode 100644 index 000000000..fed401d5c --- /dev/null +++ b/internal/commands/testdata/tas_fake/tas/migrations/.gitignore @@ -0,0 +1,2 @@ +package-lock.json +node_modules/ \ No newline at end of file diff --git a/internal/commands/testdata/tas_fake/tas/migrations/node_modules/this-file-should-be-excluded.js b/internal/commands/testdata/tas_fake/tas/migrations/node_modules/this-file-should-be-excluded.js deleted file mode 100644 index e69de29bb..000000000 diff --git a/internal/commands/testdata/tas_fake/tas/migrations/package.json b/internal/commands/testdata/tas_fake/tas/migrations/package.json new file mode 100644 index 000000000..61658a230 --- /dev/null +++ b/internal/commands/testdata/tas_fake/tas/migrations/package.json @@ -0,0 +1,15 @@ +{ + "name": "migrations", + "version": "1.0.0", + "description": "Tests for Testing Test Migrations (Test)", + "scripts": { + "test": "tap -Rspec tests/*_test.js" + }, + "author": "VMware Test Application Service (TAS)", + "license": "UNLICENSED", + "private": true, + "devDependencies": { + "tap": "^12.5.3" + }, + "dependencies": {} +} diff --git a/main.go b/main.go index 734166cf9..e1662dd83 100644 --- a/main.go +++ b/main.go @@ -85,7 +85,7 @@ func main() { } sshProvider, _ := commands.NewSshProvider(commands.SSHClientCreator{}) - commandSet["test"] = commands.NewManifestTest(outLogger, context.Background(), mobyClient, sshProvider) + commandSet["test"] = commands.NewTileTest(outLogger, context.Background(), mobyClient, sshProvider) commandSet["help"] = commands.NewHelp(os.Stdout, globalFlagsUsage, commandSet) commandSet["version"] = commands.NewVersion(outLogger, version) commandSet["update-release"] = commands.NewUpdateRelease(outLogger, fs, mrsProvider)