feat: add migrations test workflow to test command

internal/commands/manifest_test_docker/Dockerfile

@@ -23,6 +23,15 @@ RUN apt-get update && apt-get install jq -y
 # Install Ginkgo
 RUN go install github.com/onsi/ginkgo/ginkgo@latest && touch $HOME/.ack-ginkgo-rc
 
+# Install NPM
+ENV NODE_VERSION=18.16.0
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
+ENV NVM_DIR=/root/.nvm
+RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION}
+RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION}
+RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION}
+ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin:${PATH}"
+
 # Install ops-manifest
 #   assumes ops-manifest repo was cloned into ./vendor/ops-manager/gems/ops-manifest
 COPY --from=builder /tmp/ops-manager /tmp/ops-manager

internal/commands/test_tile.go

@@ -72,11 +72,13 @@ func (l infoLog) Writer() io.Writer {
 	return l.logger.Writer()
 }
 
-type ManifestTest struct {
+type TileTest struct {
 	Options struct {
 		TilePath            string `short:"tp"   long:"tile-path"                default:"."                             description:"path to the tile directory (e.g., ~/workspace/tas/ist)"`
 		GingkoManifestFlags string `short:"gmf"  long:"ginkgo-manifest-flags"    default:"-r -p -slowSpecThreshold 15"   description:"flags to pass to the gingko manifest test suite"`
 		Verbose             bool   `short:"v"    long:"verbose"                  default:"false"                         description:"log info lines. this doesn't apply to ginkgo.'"`
+		ManifestOnly        bool   `             long:"manifest-only"            default:"false"                         description:"run only manifest tests"`
+		MigrationsOnly      bool   `             long:"migrations-only"          default:"false"                         description:"run only migration tests"`
 	}
 
 	logger      *log.Logger
@@ -86,9 +88,9 @@ type ManifestTest struct {
 	sshProvider SshProvider
 }
 
-func NewManifestTest(logger *log.Logger, ctx context.Context, mobi mobyClient, sshThing SshProvider) ManifestTest {
+func NewTileTest(logger *log.Logger, ctx context.Context, mobi mobyClient, sshThing SshProvider) TileTest {
 	ctx, cancelFunc := context.WithCancel(ctx)
-	return ManifestTest{
+	return TileTest{
 		ctx:         ctx,
 		cancelFunc:  cancelFunc,
 		logger:      logger,
@@ -101,7 +103,7 @@ func NewManifestTest(logger *log.Logger, ctx context.Context, mobi mobyClient, s
 //go:embed manifest_test_docker/*
 var dockerfileContents string
 
-func (u ManifestTest) Execute(args []string) error {
+func (u TileTest) Execute(args []string) error {
 	if u.sshProvider == nil {
 		return errors.New("ssh provider failed to initialize. check your ssh-agent is running")
 	}
@@ -191,8 +193,22 @@ func (u ManifestTest) Execute(args []string) error {
 
 	loggerWithInfo.Info("Mounting", parentDir, "and testing", tileDir)
 
-	envVars := getManifestTestEnvVars(absRepoDir, tileDir)
-	dockerCmd := fmt.Sprintf("cd /tas/%s/test/manifest && PRODUCT=%s RENDERER=ops-manifest ginkgo %s", tileDir, toProduct(tileDir), u.Options.GingkoManifestFlags)
+	runAll := !u.Options.ManifestOnly && !u.Options.MigrationsOnly
+
+	var dockerCmds []string
+	if u.Options.ManifestOnly || runAll {
+		dockerCmds = append(dockerCmds, fmt.Sprintf("cd /tas/%s/test/manifest", tileDir))
+		dockerCmds = append(dockerCmds, fmt.Sprintf("PRODUCT=%s RENDERER=ops-manifest ginkgo %s", toProduct(tileDir), u.Options.GingkoManifestFlags))
+	}
+	if u.Options.MigrationsOnly || runAll {
+		dockerCmds = append(dockerCmds, fmt.Sprintf("cd /tas/%s/migrations", tileDir))
+		dockerCmds = append(dockerCmds, "npm install")
+		dockerCmds = append(dockerCmds, "npm test")
+	}
+
+	dockerCmd := strings.Join(dockerCmds, " && ")
+
+	envVars := getTileTestEnvVars(absRepoDir, tileDir)
 	loggerWithInfo.Info("Running:", dockerCmd)
 	createResp, err := u.mobi.ContainerCreate(u.ctx, &container.Config{
 		Image: "kiln_test_dependencies:vmware",
@@ -272,7 +288,7 @@ func toProduct(dir string) string {
 	}
 }
 
-func getManifestTestEnvVars(dir, productDir string) []string {
+func getTileTestEnvVars(dir, productDir string) []string {
 	const fixturesFormat = "%s/test/manifest/fixtures"
 	metadataPath := fmt.Sprintf(fixturesFormat+"/tas_metadata.yml", dir)
 	configPath := fmt.Sprintf(fixturesFormat+"/tas_config.yml", dir)
@@ -313,7 +329,7 @@ func getTarReader(fileContents string) (*bufio.Reader, error) {
 	return tr, nil
 }
 
-func (u ManifestTest) addMissingKeys() error {
+func (u TileTest) addMissingKeys() error {
 	needsKeys, err := u.sshProvider.NeedsKeys()
 	if needsKeys {
 		key, err := u.sshProvider.GetKeys()
@@ -354,7 +370,7 @@ type ErrorLine struct {
 	Error string `json:"error"`
 }
 
-func (u ManifestTest) Usage() jhanda.Usage {
+func (u TileTest) Usage() jhanda.Usage {
 	return jhanda.Usage{
 		Description:      "Test the manifest for a product inside a docker container. Requires a docker daemon to be running and ssh keys with access to Ops Manager's git repo. For non-interactive use either set the env var SSH_PASSWORD or add your ssh identify before running.",
 		ShortDescription: "Test manifest for a product",

internal/commands/test_tile_acceptance_test.go

@@ -16,7 +16,7 @@ import (
 )
 
 var _ = Describe("test", func() {
-	Context("manifest tests succeed", func() {
+	Context("all tests succeed", func() {
 		It("succeeds", func() {
 			var testOutput bytes.Buffer
 			logger := log.New(&testOutput, "", 0)
@@ -28,16 +28,17 @@ var _ = Describe("test", func() {
 			Expect(err).NotTo(HaveOccurred())
 			tilePath := filepath.Join("testdata", "tas_fake", "tas")
 			Expect(goVendor(tilePath)).NotTo(HaveOccurred())
-			testTile := commands.NewManifestTest(logger, ctx, cli, sshProvider)
+			testTile := commands.NewTileTest(logger, ctx, cli, sshProvider)
 			err = testTile.Execute([]string{"--verbose", "--tile-path", tilePath})
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(testOutput.String()).To(ContainSubstring("SUCCESS"))
+			Expect(testOutput.String()).To(ContainSubstring("hello, world"))
 			Expect(testOutput.String()).NotTo(ContainSubstring("Failure"))
 		})
 	})
 
-	Context("manifest tests fail", func() {
+	Context("all tests fail", func() {
 		It("fails", func() {
 			var testOutput bytes.Buffer
 			logger := log.New(&testOutput, "", 0)
@@ -47,7 +48,7 @@ var _ = Describe("test", func() {
 
 			sshProvider, err := commands.NewSshProvider(commands.SSHClientCreator{})
 			Expect(err).NotTo(HaveOccurred())
-			testTile := commands.NewManifestTest(logger, ctx, cli, sshProvider)
+			testTile := commands.NewTileTest(logger, ctx, cli, sshProvider)
 			tilePath := filepath.Join("testdata", "tas_fake", "tas_failing")
 			Expect(goVendor(tilePath)).NotTo(HaveOccurred())
 			err = testTile.Execute([]string{"--verbose", "--tile-path", tilePath})

internal/commands/test_tile_test.go

@@ -47,7 +47,7 @@ var _ = Describe("kiln test docker", func() {
 			logger = log.New(&writer, "", 0)
 		})
 
-		Describe("successful creation creation", func() {
+		Describe("test outcomes", func() {
 			var (
 				fakeSshProvider *fakes.SshProvider
 				helloTilePath   string
@@ -73,15 +73,15 @@ var _ = Describe("kiln test docker", func() {
 				})
 				When("executing tests", func() {
 					var (
-						subjectUnderTest commands.ManifestTest
+						subjectUnderTest commands.TileTest
 					)
 					BeforeEach(func() {
 						writer.Reset()
-						subjectUnderTest = commands.NewManifestTest(logger, ctx, fakeMobyClient, fakeSshProvider)
+						subjectUnderTest = commands.NewTileTest(logger, ctx, fakeMobyClient, fakeSshProvider)
 					})
 					When("verbose is passed", func() {
 						It("succeeds and logs info", func() {
-							err := subjectUnderTest.Execute([]string{"--verbose", "--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"})
+							err := subjectUnderTest.Execute([]string{"--manifest-only", "--verbose", "--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"})
 							Expect(err).To(BeNil())
 
 							By("logging helpful messages", func() {
@@ -112,7 +112,7 @@ var _ = Describe("kiln test docker", func() {
 					})
 					When("verbose isn't passed", func() {
 						It("doesn't log info", func() {
-							err := subjectUnderTest.Execute([]string{"--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"})
+							err := subjectUnderTest.Execute([]string{"--manifest-only", "--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"})
 							Expect(err).To(BeNil())
 
 							By("logging helpful messages", func() {
@@ -141,8 +141,8 @@ var _ = Describe("kiln test docker", func() {
 					fakeMobyClient = setupFakeMobyClient(testFailureMessage, 1)
 				})
 				It("returns an error", func() {
-					subjectUnderTest := commands.NewManifestTest(logger, ctx, fakeMobyClient, fakeSshProvider)
-					err := subjectUnderTest.Execute([]string{"--verbose", "--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"})
+					subjectUnderTest := commands.NewTileTest(logger, ctx, fakeMobyClient, fakeSshProvider)
+					err := subjectUnderTest.Execute([]string{"--manifest-only", "--verbose", "--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"})
 					Expect(err).To(HaveOccurred())
 
 					By("logging helpful messages", func() {
@@ -153,14 +153,94 @@ var _ = Describe("kiln test docker", func() {
 					})
 				})
 			})
+
+			When("all tests are run", func() {
+				var (
+					fakeMobyClient *fakes.MobyClient
+				)
+				BeforeEach(func() {
+					fakeMobyClient = setupFakeMobyClient("success", 0)
+				})
+				When("executing migration tests", func() {
+					var (
+						subjectUnderTest commands.TileTest
+					)
+					BeforeEach(func() {
+						writer.Reset()
+						subjectUnderTest = commands.NewTileTest(logger, ctx, fakeMobyClient, fakeSshProvider)
+					})
+
+					It("succeeds", func() {
+						err := subjectUnderTest.Execute([]string{"--tile-path", helloTilePath})
+						Expect(err).To(BeNil())
+
+						By("creating a test container", func() {
+							Expect(fakeMobyClient.ContainerCreateCallCount()).To(Equal(1))
+							_, config, _, _, _, _ := fakeMobyClient.ContainerCreateArgsForCall(0)
+
+							By("executing the tests", func() {
+								dockerCmd := "cd /tas/hello-tile/test/manifest && PRODUCT=hello-tile RENDERER=ops-manifest ginkgo -r -p -slowSpecThreshold 15 && cd /tas/hello-tile/migrations && npm install && npm test"
+								Expect(config.Cmd).To(Equal(strslice.StrSlice{"/bin/bash", "-c", dockerCmd}))
+							})
+						})
+					})
+				})
+			})
+
+			When("migration tests should be successful", func() {
+				const (
+					testSuccessLogLine = "migration tests completed successfully"
+				)
+				var (
+					fakeMobyClient *fakes.MobyClient
+				)
+				BeforeEach(func() {
+					fakeMobyClient = setupFakeMobyClient(testSuccessLogLine, 0)
+				})
+				When("executing migration tests", func() {
+					var (
+						subjectUnderTest commands.TileTest
+					)
+					BeforeEach(func() {
+						writer.Reset()
+						subjectUnderTest = commands.NewTileTest(logger, ctx, fakeMobyClient, fakeSshProvider)
+					})
+
+					It("succeeds and logs info", func() {
+						err := subjectUnderTest.Execute([]string{"--migrations-only", "--verbose", "--tile-path", helloTilePath})
+						Expect(err).To(BeNil())
+
+						By("logging helpful messages", func() {
+							logs := writer.String()
+							By("logging container information", func() {
+								Expect(logs).To(ContainSubstring("Building / restoring cached docker image"))
+							})
+							By("logging test lines", func() {
+								Expect(logs).To(ContainSubstring("migration tests completed successfully"))
+							})
+						})
+
+						By("creating a test container", func() {
+							Expect(fakeMobyClient.ContainerCreateCallCount()).To(Equal(1))
+							_, config, _, _, _, _ := fakeMobyClient.ContainerCreateArgsForCall(0)
+
+							By("executing the tests", func() {
+								dockerCmd := "cd /tas/hello-tile/migrations && npm install && npm test"
+								Expect(config.Cmd).To(Equal(strslice.StrSlice{"/bin/bash", "-c", dockerCmd}))
+							})
+						})
+					})
+				})
+			})
+
 		})
 
 		It("exits with an error if docker isn't running", func() {
 			fakeMobyClient := &fakes.MobyClient{}
 			fakeMobyClient.PingReturns(types.Ping{}, errors.New("docker not running"))
 			fakeSshThinger := fakes.SshProvider{}
 			fakeSshThinger.NeedsKeysReturns(false, nil)
-			subjectUnderTest := commands.NewManifestTest(logger, ctx, fakeMobyClient, &fakeSshThinger)
+			subjectUnderTest := commands.NewTileTest(logger, ctx, fakeMobyClient, &fakeSshThinger)
 			err := subjectUnderTest.Execute([]string{filepath.Join(helloTileDirectorySegments...)})
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("Docker daemon is not running"))

internal/commands/testdata/tas_fake/tas/migrations/.gitignore

@@ -0,0 +1,2 @@
+package-lock.json
+node_modules/

internal/commands/testdata/tas_fake/tas/migrations/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "migrations",
+  "version": "1.0.0",
+  "description": "Tests for Testing Test Migrations (Test)",
+  "scripts": {
+    "test": "tap -Rspec tests/*_test.js"
+  },
+  "author": "VMware Test Application Service (TAS)",
+  "license": "UNLICENSED",
+  "private": true,
+  "devDependencies": {
+    "tap": "^12.5.3"
+  },
+  "dependencies": {}
+}

main.go

@@ -85,7 +85,7 @@ func main() {
 	}
 
 	sshProvider, _ := commands.NewSshProvider(commands.SSHClientCreator{})
-	commandSet["test"] = commands.NewManifestTest(outLogger, context.Background(), mobyClient, sshProvider)
+	commandSet["test"] = commands.NewTileTest(outLogger, context.Background(), mobyClient, sshProvider)
 	commandSet["help"] = commands.NewHelp(os.Stdout, globalFlagsUsage, commandSet)
 	commandSet["version"] = commands.NewVersion(outLogger, version)
 	commandSet["update-release"] = commands.NewUpdateRelease(outLogger, fs, mrsProvider)