Support using verbose makefile in mbt build + allow to set number of …

…jobs (#577)

* Support using verbose makefile in mbt build + allow to set number of jobs
* Add message when starting and finishing module build

cmd/cmd_test.go

@@ -1,32 +1,26 @@
 package commands
 
 import (
-	dir "github.com/SAP/cloud-mta-build-tool/internal/archive"
 	"os"
 	"path/filepath"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	dir "github.com/SAP/cloud-mta-build-tool/internal/archive"
 	"github.com/SAP/cloud-mta-build-tool/internal/logs"
 )
 
 var _ = Describe("Commands", func() {
 
 	BeforeEach(func() {
-		mtadCmdTrg = getTestPath("result")
-		metaCmdTrg = getTestPath("result")
-		mtarCmdTrg = getTestPath("result")
-		packCmdTrg = getTestPath("result")
-		buildCmdTrg = getTestPath("result")
 		cleanupCmdTrg = getTestPath("result")
 		logs.Logger = logs.NewLogger()
-		err := dir.CreateDirIfNotExist(mtadCmdTrg)
-		Ω(err).Should(Succeed())
+		Ω(dir.CreateDirIfNotExist(cleanupCmdTrg)).Should(Succeed())
 	})
 
 	AfterEach(func() {
-		os.RemoveAll(mtadCmdTrg)
+		Ω(os.RemoveAll(cleanupCmdTrg)).Should(Succeed())
 	})
 
 	var _ = Describe("cleanup command", func() {

cmd/init.go

@@ -1,6 +1,7 @@
 package commands
 
 import (
+	"fmt"
 	"os"
 	"time"
 
@@ -22,12 +23,14 @@ var initCmdExtensions []string
 var initCmdMode string
 
 // flags of build command
-var buildProjectCmdSrc string
-var buildProjectCmdTrg string
-var buildProjectCmdExtensions []string
-var buildProjectCmdMtar = "*"
-var buildProjectCmdPlatform string
-var buildProjectCmdStrict bool
+var buildCmdSrc string
+var buildCmdTrg string
+var buildCmdExtensions []string
+var buildCmdMtar = "*"
+var buildCmdPlatform string
+var buildCmdStrict bool
+var buildCmdMode string
+var buildCmdJobs int
 
 func init() {
 	// set flags for init command
@@ -39,12 +42,16 @@ func init() {
 	initCmd.Flags().BoolP("help", "h", false, `Displays detailed information about the "init" command`)
 
 	// set flags of build command
-	buildCmd.Flags().StringVarP(&buildProjectCmdSrc, "source", "s", "", "The path to the MTA project; the current path is set as the default")
-	buildCmd.Flags().StringVarP(&buildProjectCmdTrg, "target", "t", "", "The path to the results folder; the current path is set as the default")
-	buildCmd.Flags().StringSliceVarP(&buildProjectCmdExtensions, "extensions", "e", nil, "The MTA extension descriptors")
-	buildCmd.Flags().StringVarP(&buildProjectCmdMtar, "mtar", "", "", "The file name of the generated archive file")
-	buildCmd.Flags().StringVarP(&buildProjectCmdPlatform, "platform", "p", "cf", `The deployment platform; supported platforms: "cf", "xsa", "neo"`)
-	buildCmd.Flags().BoolVarP(&buildProjectCmdStrict, "strict", "", true, `If set to true, duplicated fields and fields not defined in the "mta.yaml" schema are reported as errors; if set to false, they are reported as warnings`)
+	buildCmd.Flags().StringVarP(&buildCmdSrc, "source", "s", "", "The path to the MTA project; the current path is set as the default")
+	buildCmd.Flags().StringVarP(&buildCmdTrg, "target", "t", "", "The path to the results folder; the current path is set as the default")
+	buildCmd.Flags().StringSliceVarP(&buildCmdExtensions, "extensions", "e", nil, "The MTA extension descriptors")
+	buildCmd.Flags().StringVarP(&buildCmdMtar, "mtar", "", "", "The file name of the generated archive file")
+	buildCmd.Flags().StringVarP(&buildCmdPlatform, "platform", "p", "cf", `The deployment platform; supported platforms: "cf", "xsa", "neo"`)
+	buildCmd.Flags().BoolVarP(&buildCmdStrict, "strict", "", true, `If set to true, duplicated fields and fields not defined in the "mta.yaml" schema are reported as errors; if set to false, they are reported as warnings`)
+	buildCmd.Flags().StringVarP(&buildCmdMode, "mode", "m", "", `(beta) The mode of the Makefile generation; supported values: "default" and "verbose"`)
+	_ = buildCmd.Flags().MarkHidden("mode")
+	buildCmd.Flags().IntVarP(&buildCmdJobs, "jobs", "j", 0, fmt.Sprintf(`The number of Make recipes to be executed simultaneously when building the project; if 0 or less, the number of available CPUs (limited to %d) is used. Used only in verbose mode.`, artifacts.MaxMakeParallel))
+	_ = buildCmd.Flags().MarkHidden("jobs")
 	buildCmd.Flags().BoolP("help", "h", false, `Displays detailed information about the "build" command`)
 }
 
@@ -71,7 +78,7 @@ var buildCmd = &cobra.Command{
 		makefileTmp := "Makefile_" + time.Now().Format("20060102150405") + ".mta"
 		// Generate build script
 		// Note: we can only use the non-default mbt (i.e. the current executable name) from inside the command itself because if this function runs from other places like tests it won't point to the MBT
-		err := artifacts.ExecBuild(makefileTmp, buildProjectCmdSrc, buildProjectCmdTrg, buildProjectCmdExtensions, "", buildProjectCmdMtar, buildProjectCmdPlatform, buildProjectCmdStrict, os.Getwd, exec.Execute, false)
+		err := artifacts.ExecBuild(makefileTmp, buildCmdSrc, buildCmdTrg, buildCmdExtensions, buildCmdMode, buildCmdMtar, buildCmdPlatform, buildCmdStrict, buildCmdJobs, os.Getwd, exec.Execute, false)
 		return err
 	},
 	SilenceUsage: true,

cmd/init_test.go

@@ -42,9 +42,9 @@ var _ = Describe("Build", func() {
 		}
 	})
 	It("Failure - wrong platform", func() {
-		buildProjectCmdSrc = getTestPath("mta")
-		buildProjectCmdTrg = getTestPath("result")
-		buildProjectCmdPlatform = "xxx"
+		buildCmdSrc = getTestPath("mta")
+		buildCmdTrg = getTestPath("result")
+		buildCmdPlatform = "xxx"
 		err := buildCmd.RunE(nil, []string{})
 		Ω(err).Should(HaveOccurred())
 	})

cmd/module.go

@@ -16,11 +16,11 @@ var packCmdModule string
 var packCmdPlatform string
 
 // flags of build command
-var buildCmdSrc string
-var buildCmdTrg string
-var buildCmdExtensions []string
-var buildCmdModule string
-var buildCmdPlatform string
+var buildModuleCmdSrc string
+var buildModuleCmdTrg string
+var buildModuleCmdExtensions []string
+var buildModuleCmdModule string
+var buildModuleCmdPlatform string
 
 func init() {
 
@@ -37,15 +37,15 @@ func init() {
 		`The deployment platform; supported platforms: "cf", "xsa", "neo"`)
 
 	// sets the flags of the command build module
-	buildModuleCmd.Flags().StringVarP(&buildCmdSrc, "source", "s", "",
+	buildModuleCmd.Flags().StringVarP(&buildModuleCmdSrc, "source", "s", "",
 		"The path to the MTA project; the current path is set as the default")
-	buildModuleCmd.Flags().StringVarP(&buildCmdTrg, "target", "t", "",
+	buildModuleCmd.Flags().StringVarP(&buildModuleCmdTrg, "target", "t", "",
 		"The path to the results folder; the current path is set as the default")
-	buildModuleCmd.Flags().StringSliceVarP(&buildCmdExtensions, "extensions", "e", nil,
+	buildModuleCmd.Flags().StringSliceVarP(&buildModuleCmdExtensions, "extensions", "e", nil,
 		"The MTA extension descriptors")
-	buildModuleCmd.Flags().StringVarP(&buildCmdModule, "module", "m", "",
+	buildModuleCmd.Flags().StringVarP(&buildModuleCmdModule, "module", "m", "",
 		"The name of the module")
-	buildModuleCmd.Flags().StringVarP(&buildCmdPlatform, "platform", "p", "cf",
+	buildModuleCmd.Flags().StringVarP(&buildModuleCmdPlatform, "platform", "p", "cf",
 		`The deployment platform; supported platforms: "cf", "xsa", "neo"`)
 }
 
@@ -56,7 +56,7 @@ var buildModuleCmd = &cobra.Command{
 	Long:  "Builds module and archives its artifacts",
 	Args:  cobra.NoArgs,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		err := artifacts.ExecuteBuild(buildCmdSrc, buildCmdTrg, buildCmdExtensions, buildCmdModule, buildCmdPlatform, os.Getwd)
+		err := artifacts.ExecuteBuild(buildModuleCmdSrc, buildModuleCmdTrg, buildModuleCmdExtensions, buildModuleCmdModule, buildModuleCmdPlatform, os.Getwd)
 		logError(err)
 		return err
 	},

cmd/module_test.go

@@ -23,7 +23,7 @@ var _ = Describe("Commands", func() {
 		metaCmdTrg = getTestPath("result")
 		mtarCmdTrg = getTestPath("result")
 		packCmdTrg = getTestPath("result")
-		buildCmdTrg = getTestPath("result")
+		buildModuleCmdTrg = getTestPath("result")
 		cleanupCmdTrg = getTestPath("result")
 		logs.Logger = logs.NewLogger()
 		err := os.Mkdir(mtadCmdTrg, os.ModePerm)
@@ -98,12 +98,12 @@ builders:
 		})
 
 		It("build Command", func() {
-			buildCmdModule = "node-js"
-			buildCmdSrc = getTestPath("mta")
-			buildCmdPlatform = "cf"
-			ep := dir.Loc{SourcePath: buildCmdSrc, TargetPath: buildCmdTrg}
+			buildModuleCmdModule = "node-js"
+			buildModuleCmdSrc = getTestPath("mta")
+			buildModuleCmdPlatform = "cf"
+			ep := dir.Loc{SourcePath: buildModuleCmdSrc, TargetPath: buildModuleCmdTrg}
 			Ω(buildModuleCmd.RunE(nil, []string{})).Should(Succeed())
-			Ω(ep.GetTargetModuleZipPath(buildCmdModule)).Should(BeAnExistingFile())
+			Ω(ep.GetTargetModuleZipPath(buildModuleCmdModule)).Should(BeAnExistingFile())
 		})
 	})
 })

integration/cloud_mta_build_tool_test.go

@@ -18,6 +18,7 @@ import (
 	"time"
 
 	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/extensions/table"
 	. "github.com/onsi/gomega"
 
 	dir "github.com/SAP/cloud-mta-build-tool/internal/archive"
@@ -375,36 +376,73 @@ modules:
 			Ω(filepath.Join(dir, "testdata", "mta_demo", "mta_archives", demoArchiveName)).Should(BeAnExistingFile())
 		})
 
-		It("Generate Verbose Makefile with module dependencies", func() {
+		Describe("cleanup after tests", func() {
 			dir, _ := os.Getwd()
 			path := filepath.Join(dir, "testdata", "moduledep")
 			archivePath := filepath.Join(path, "mta_archives", "f1_0.0.1.mtar")
 			tempZipPath := filepath.Join(path, "mta_archives", "data.zip")
 
-			Ω(os.RemoveAll(filepath.Join(path, "Makefile.mta"))).Should(Succeed())
-			Ω(os.RemoveAll(archivePath)).Should(Succeed())
+			AfterEach(func() {
+				Ω(os.RemoveAll(filepath.Join(path, "Makefile.mta"))).Should(Succeed())
+				Ω(os.RemoveAll(filepath.Join(path, "mta_archives"))).Should(Succeed())
+				Ω(os.RemoveAll(filepath.Join(path, "public", "client"))).Should(Succeed())
+				Ω(os.RemoveAll(filepath.Join(path, "public", "client2"))).Should(Succeed())
+			})
+
+			It("Generate Verbose Makefile with module dependencies", func() {
+				bin := filepath.FromSlash(binPath)
+				cmdOut, errString, _ := execute(bin, "init -m=verbose", path)
+				Ω(errString).Should(Equal(""))
+				Ω(cmdOut).ShouldNot(BeNil())
+				// Check the MakeFile was generated
+				Ω(filepath.Join(path, "Makefile.mta")).Should(BeAnExistingFile())
+
+				// Generate mtar
+				bin = filepath.FromSlash("make")
+				execute(bin, "-f Makefile.mta p=cf", path)
+				// Check the archive was generated
+				Ω(archivePath).Should(BeAnExistingFile())
+				validateMtaArchiveContents([]string{"module_with_dep/", "module_with_dep/data.zip"}, archivePath)
+
+				// Extract data.zip and check its content
+				err := extractFileFromZip(archivePath, "module_with_dep/data.zip", tempZipPath)
+				Ω(err).Should(Succeed())
+				validateArchiveContents([]string{"package.json", "client/", "client/client_package.json", "client2/", "client2/client_package.json"}, tempZipPath)
+			})
+		})
+	})
+
+	Describe("cleanup after tests", func() {
+		dir, _ := os.Getwd()
+		path := filepath.Join(dir, "testdata", "moduledep")
+		archivePath := filepath.Join(path, "mta_archives", "f1_0.0.1.mtar")
+		tempZipPath := filepath.Join(path, "mta_archives", "data.zip")
+
+		AfterEach(func() {
+			Ω(os.RemoveAll(filepath.Join(path, "mta_archives"))).Should(Succeed())
 			Ω(os.RemoveAll(filepath.Join(path, "public", "client"))).Should(Succeed())
-			Ω(os.RemoveAll(tempZipPath)).Should(Succeed())
+			Ω(os.RemoveAll(filepath.Join(path, "public", "client2"))).Should(Succeed())
+		})
 
+		DescribeTable("Build MTA with module dependencies", func(additionalBuildOpts []string) {
 			bin := filepath.FromSlash(binPath)
-			cmdOut, errString, _ := execute(bin, "init -m=verbose", path)
+			cmdOut, errString, _ := executeWithArgs(bin, path, append([]string{"build"}, additionalBuildOpts...)...)
 			Ω(errString).Should(Equal(""))
 			Ω(cmdOut).ShouldNot(BeNil())
-			// Check the MakeFile was generated
-			Ω(filepath.Join(path, "Makefile.mta")).Should(BeAnExistingFile())
 
-			// Generate mtar
-			bin = filepath.FromSlash("make")
-			execute(bin, "-f Makefile.mta p=cf", path)
 			// Check the archive was generated
 			Ω(archivePath).Should(BeAnExistingFile())
 			validateMtaArchiveContents([]string{"module_with_dep/", "module_with_dep/data.zip"}, archivePath)
 
 			// Extract data.zip and check its content
 			err := extractFileFromZip(archivePath, "module_with_dep/data.zip", tempZipPath)
 			Ω(err).Should(Succeed())
-			validateArchiveContents([]string{"package.json", "client/", "client/client_package.json"}, tempZipPath)
-		})
+			validateArchiveContents([]string{"package.json", "client/", "client/client_package.json", "client2/", "client2/client_package.json"}, tempZipPath)
+		},
+			Entry("Non-verbose build", []string{}),
+			Entry("Parallel verbose build", []string{"--mode=verbose"}),
+			Entry("Serial verbose build", []string{"--mode=verbose", "--jobs=1"}),
+		)
 	})
 
 	var _ = Describe("MBT gen commands", func() {
@@ -731,8 +769,12 @@ func executeEverySecond(bin string, args string, path string) (string, error str
 
 // Execute commands and get outputs
 func execute(bin string, args string, path string) (output string, error string, cmd *exec.Cmd) {
+	return executeWithArgs(bin, path, strings.Split(args, " ")...)
+}
+
+func executeWithArgs(bin string, path string, args ...string) (output string, error string, cmd *exec.Cmd)  {
 	// Provide list of commands
-	cmd = exec.Command(bin, strings.Split(args, " ")...)
+	cmd = exec.Command(bin, args...)
 	// bin path
 	cmd.Dir = path
 	// std out
@@ -751,4 +793,4 @@ func execute(bin string, args string, path string) (output string, error string,
 		fmt.Println(err)
 	}
 	return stdoutBuf.String(), stdErrBuf.String(), cmd
-}
+}

integration/testdata/moduledep/client2/client_package.json

@@ -0,0 +1,34 @@
+{
+	"name": "f1.client",
+	"title": "f1",
+	"version": "0.0.1",
+	"description": "",
+	"scripts": {
+		"postinstall": "node ./scripts/bundle.js && npm run lint",
+		"lint": "node ./node_modules/eslint/bin/eslint . --ext=.js,.json"
+	},
+	"devDependencies": {
+		"eslint": "4.12.1",
+		"@sap/eslint-plugin-webide-feature": "1.3.15",
+		"@sap-webide/webide-client-tools": "latest"
+	},
+	"bundledPlugins": {
+		"p1": "file:src/p1"
+	},
+	"files": [
+		"i18n",
+		"src",
+		"config-preload.js",
+		"config-preload.json",
+		"package.json"
+	],
+	"webidePreloads": {
+		"js": [
+			"config-preload.js",
+			"i18n/config-preload.js"
+		],
+		"config": [
+			"config-preload.json"
+		]
+	}
+}

integration/testdata/moduledep/mta.yaml

@@ -12,10 +12,18 @@ modules:
         - name: m2
           artifacts: ["*"]
           target-path: "client"
+        - name: m3
+          artifacts: ["*"]
+          target-path: "client2"
   - name: m2
     type: html5
     path: client
     build-parameters:
       builder: zip
       supported-platforms: []
-
+  - name: m3
+    type: html5
+    path: client2
+    build-parameters:
+      builder: zip
+      supported-platforms: []

internal/artifacts/artifacts_msg.go

@@ -34,6 +34,7 @@ const (
 	genMTARArchMsg    = `could not generate the MTA archive when archiving`
 
 	buildMsg                 = `building the "%s" module...`
+	buildFinishedMsg         = `finished building the "%s" module`
 	buildFailedMsg           = `could not build the "%s" module`
 	buildFailedOnCommandsMsg = `could not get commands for the "%s" module`
 	buildFailedOnDepsMsg     = `could not process dependencies for the "%s" module`

internal/artifacts/module_arch.go

@@ -36,6 +36,7 @@ func ExecuteBuild(source, target string, extensions []string, moduleName, platfo
 	if err != nil {
 		return err
 	}
+	logs.Logger.Infof(buildFinishedMsg, moduleName)
 	return nil
 }