Support build dependencies between modules in verbose makefile (#551)

cmd/cmd.go

@@ -25,7 +25,7 @@ var validateCmdExclude string
 func init() {
 
 	// Add command to the root
-	rootCmd.AddCommand(initCmd, buildCmd, validateCmd, cleanupCmd, provideCmd, generateCmd, moduleCmd, assembleCommand, projectCmd, mergeCmd, executeCommand)
+	rootCmd.AddCommand(initCmd, buildCmd, validateCmd, cleanupCmd, provideCmd, generateCmd, moduleCmd, assembleCommand, projectCmd, mergeCmd, executeCommand, copyCmd)
 	// Build module
 	provideCmd.AddCommand(provideModuleCmd)
 	// generate immutable commands

cmd/exec.go

@@ -3,11 +3,15 @@ package commands
 import (
 	"github.com/spf13/cobra"
 
+	"github.com/SAP/cloud-mta-build-tool/internal/archive"
 	"github.com/SAP/cloud-mta-build-tool/internal/exec"
 )
 
 var executeCmdCommands []string
 var executeCmdTimeout string
+var copyCmdSrc string
+var copyCmdTrg string
+var copyCmdPatterns []string
 
 // Execute commands in the current working directory with a timeout.
 // This is used in verbose make files for implementing a timeout on module builds.
@@ -26,10 +30,35 @@ var executeCommand = &cobra.Command{
 	SilenceErrors: true,
 }
 
+// Copy files matching the specified patterns from the source path to the target path.
+// This is used in verbose make files for copying artifacts from a module's dependencies before building the module.
+var copyCmd = &cobra.Command{
+	Use:   "cp",
+	Short: "Copy files by patterns",
+	Long:  "Copy files by patterns",
+	Args:  cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		err := dir.CopyByPatterns(copyCmdSrc, copyCmdTrg, copyCmdPatterns)
+		logError(err)
+		return err
+	},
+	SilenceUsage:  true,
+	Hidden:        true,
+	SilenceErrors: true,
+}
+
 func init() {
 	// set flag of execute command
 	executeCommand.Flags().StringArrayVarP(&executeCmdCommands,
 		"commands", "c", nil, "commands to run")
 	executeCommand.Flags().StringVarP(&executeCmdTimeout,
 		"timeout", "t", "", "the timeout after which the run stops, in the format [123h][123m][123s]; 10m is set as the default")
+
+	// set flags of copy command
+	copyCmd.Flags().StringVarP(&copyCmdSrc, "source", "s", "",
+		"the path to the source folder")
+	copyCmd.Flags().StringVarP(&copyCmdTrg, "target", "t", "",
+		"the path to the target folder")
+	copyCmd.Flags().StringArrayVarP(&copyCmdPatterns,
+		"patterns", "p", nil, "patterns for matching the files and folders to copy")
 }

cmd/exec_test.go

@@ -0,0 +1,93 @@
+package commands
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/SAP/cloud-mta-build-tool/internal/archive"
+)
+
+var _ = Describe("cp command", func() {
+	BeforeEach(func() {
+		copyCmdSrc = getTestPath("mtahtml5")
+		copyCmdTrg = getTestPath("result")
+		copyCmdPatterns = []string{}
+		err := dir.CreateDirIfNotExist(copyCmdTrg)
+		Ω(err).Should(Succeed())
+	})
+
+	AfterEach(func() {
+		Ω(os.RemoveAll(getTestPath("result"))).Should(Succeed())
+	})
+
+	It("copy - Sanity", func() {
+		copyCmdPatterns = []string{"mta.*", "ui5app2"}
+		Ω(copyCmd.RunE(nil, []string{})).Should(Succeed())
+		validateFilesInDir(getTestPath("result"), []string{"mta.sh", "mta.yaml", "ui5app2/", "ui5app2/test.txt"})
+	})
+	It("copy should not fail when the pattern is valid but doesn't match any files", func() {
+		copyCmdPatterns = []string{"xxx.xxx"}
+		Ω(copyCmd.RunE(nil, []string{})).Should(Succeed())
+		validateFilesInDir(getTestPath("result"), []string{})
+	})
+	It("copy should not fail when there are no patterns", func() {
+		copyCmdPatterns = []string{}
+		Ω(copyCmd.RunE(nil, []string{})).Should(Succeed())
+		validateFilesInDir(getTestPath("result"), []string{})
+	})
+	It("copy should return error when the source folder doesn't exist", func() {
+		copyCmdSrc = getTestPath("mtahtml6")
+		copyCmdPatterns = []string{"*"}
+		Ω(copyCmd.RunE(nil, []string{})).Should(HaveOccurred())
+	})
+	It("copy should return error when a pattern is invalid", func() {
+		copyCmdPatterns = []string{"["}
+		Ω(copyCmd.RunE(nil, []string{})).Should(HaveOccurred())
+	})
+})
+
+// Check the folder exists and includes exactly the expected files
+func validateFilesInDir(src string, expectedFilesInDir []string) {
+	// List all files in the folder recursively
+	filesInDir := make([]string, 0)
+	err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		// Don't include the folder itself
+		if filepath.Clean(path) == filepath.Clean(src) {
+			return nil
+		}
+		relPath, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+		relPath = filepath.ToSlash(relPath)
+		if info.IsDir() {
+			relPath += "/"
+		}
+		filesInDir = append(filesInDir, relPath)
+		return nil
+	})
+	Ω(err).Should(Succeed())
+
+	for _, expectedFile := range expectedFilesInDir {
+		Ω(contains(expectedFile, filesInDir)).Should(BeTrue(), fmt.Sprintf("expected %s to be in the directory; directory contains %v", expectedFile, filesInDir))
+	}
+	for _, existingFile := range filesInDir {
+		Ω(contains(existingFile, expectedFilesInDir)).Should(BeTrue(), fmt.Sprintf("did not expect %s to be in the directory; directory contains %v", existingFile, filesInDir))
+	}
+}
+
+func contains(element string, elements []string) bool {
+	for _, el := range elements {
+		if el == element {
+			return true
+		}
+	}
+	return false
+}

integration/cloud_mta_build_tool_test.go

@@ -20,6 +20,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	dir "github.com/SAP/cloud-mta-build-tool/internal/archive"
 	"github.com/SAP/cloud-mta/mta"
 )
 
@@ -374,6 +375,36 @@ modules:
 			Ω(filepath.Join(dir, "testdata", "mta_demo", "mta_archives", demoArchiveName)).Should(BeAnExistingFile())
 		})
 
+		It("Generate Verbose Makefile with module dependencies", func() {
+			dir, _ := os.Getwd()
+			path := filepath.Join(dir, "testdata", "moduledep")
+			archivePath := filepath.Join(path, "mta_archives", "f1_0.0.1.mtar")
+			tempZipPath := filepath.Join(path, "mta_archives", "data.zip")
+
+			Ω(os.RemoveAll(filepath.Join(path, "Makefile.mta"))).Should(Succeed())
+			Ω(os.RemoveAll(archivePath)).Should(Succeed())
+			Ω(os.RemoveAll(filepath.Join(path, "public", "client"))).Should(Succeed())
+			Ω(os.RemoveAll(tempZipPath)).Should(Succeed())
+
+			bin := filepath.FromSlash(binPath)
+			cmdOut, errString, _ := execute(bin, "init -m=verbose", path)
+			Ω(errString).Should(Equal(""))
+			Ω(cmdOut).ShouldNot(BeNil())
+			// Check the MakeFile was generated
+			Ω(filepath.Join(path, "Makefile.mta")).Should(BeAnExistingFile())
+
+			// Generate mtar
+			bin = filepath.FromSlash("make")
+			execute(bin, "-f Makefile.mta p=cf", path)
+			// Check the archive was generated
+			Ω(archivePath).Should(BeAnExistingFile())
+			validateMtaArchiveContents([]string{"module_with_dep/", "module_with_dep/data.zip"}, archivePath)
+
+			// Extract data.zip and check its content
+			err := extractFileFromZip(archivePath, "module_with_dep/data.zip", tempZipPath)
+			Ω(err).Should(Succeed())
+			validateArchiveContents([]string{"package.json", "client/", "client/client_package.json"}, tempZipPath)
+		})
 	})
 
 	var _ = Describe("MBT gen commands", func() {
@@ -586,13 +617,48 @@ func getFileContentFromZip(path string, filename string) ([]byte, error) {
 	return nil, fmt.Errorf(`file "%s" not found`, filename)
 }
 
+func extractFileFromZip(archivePath string, filename string, dst string) (error) {
+	zipFile, err := zip.OpenReader(archivePath)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = zipFile.Close()
+	}()
+	var fileToExtract *zip.File = nil
+	for _, file := range zipFile.File {
+		if strings.Contains(file.Name, filename) {
+			fileToExtract = file
+		}
+	}
+	if fileToExtract == nil {
+		return fmt.Errorf(`file "%s" not found`, filename)
+	}
+
+	in, err := fileToExtract.Open()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = in.Close()
+	}()
+	err = dir.WriteFile(in, dst)
+	return err
+}
+
 func validateMtaArchiveContents(expectedAdditionalFilesInArchive []string, archiveLocation string) {
 	expectedFilesInArchive := append(expectedAdditionalFilesInArchive, "META-INF/", "META-INF/MANIFEST.MF", "META-INF/mtad.yaml")
-	archiveReader, err := zip.OpenReader(archiveLocation)
+	validateArchiveContents(expectedFilesInArchive, archiveLocation)
+}
+
+func validateArchiveContents(expectedFilesInArchive []string, archiveLocation string) {
+	archive, err := zip.OpenReader(archiveLocation)
 	Ω(err).Should(Succeed())
-	defer archiveReader.Close()
+	defer func() {
+		_ = archive.Close()
+	}()
 	var filesInArchive []string
-	for _, file := range archiveReader.File {
+	for _, file := range archive.File {
 		filesInArchive = append(filesInArchive, file.Name)
 	}
 	for _, expectedFile := range expectedFilesInArchive {

integration/testdata/moduledep/client/client_package.json

@@ -0,0 +1,34 @@
+{
+	"name": "f1.client",
+	"title": "f1",
+	"version": "0.0.1",
+	"description": "",
+	"scripts": {
+		"postinstall": "node ./scripts/bundle.js && npm run lint",
+		"lint": "node ./node_modules/eslint/bin/eslint . --ext=.js,.json"
+	},
+	"devDependencies": {
+		"eslint": "4.12.1",
+		"@sap/eslint-plugin-webide-feature": "1.3.15",
+		"@sap-webide/webide-client-tools": "latest"
+	},
+	"bundledPlugins": {
+		"p1": "file:src/p1"
+	},
+	"files": [
+		"i18n",
+		"src",
+		"config-preload.js",
+		"config-preload.json",
+		"package.json"
+	],
+	"webidePreloads": {
+		"js": [
+			"config-preload.js",
+			"i18n/config-preload.js"
+		],
+		"config": [
+			"config-preload.json"
+		]
+	}
+}

integration/testdata/moduledep/mta.yaml

@@ -0,0 +1,21 @@
+_schema-version: "2.0.0"
+ID: f1
+version: 0.0.1
+
+modules:
+  - name: module_with_dep
+    type: html5
+    path: public
+    build-parameters:
+      builder: zip
+      requires:
+        - name: m2
+          artifacts: ["*"]
+          target-path: "client"
+  - name: m2
+    type: html5
+    path: client
+    build-parameters:
+      builder: zip
+      supported-platforms: []
+

integration/testdata/moduledep/public/package.json

@@ -0,0 +1,10 @@
+{
+	"name": "f1.approuter",
+	"version": "0.0.1",
+	"scripts": {
+		"start": "node node_modules/@sap/approuter/approuter.js"
+	},
+	"dependencies": {
+		"@sap/approuter": "2.7.0"
+	}
+}

internal/archive/archive_msg.go

@@ -11,6 +11,7 @@ const (
 	copyByPatternFailedOnCreateMsg   = `could not copy files matching the patterns [%s,...] from the "%s" folder to the "%s" folder: could not create the "%s" folder`
 	copyByPatternFailedOnTargetMsg   = `could not copy files matching the patterns [%s,...] from the "%s" folder to the "%s" folder: "%s" is not a folder`
 	copyByPatternFailedOnMatchMsg    = `could not copy files matching the "%s" pattern from the "%s" folder to the "%s": could not get list of files matching the "%s" pattern`
+	wrongPathMsg                     = `could not find the "%s" path`
 
 	// InitLocFailedOnWorkDirMsg - message raised on getting working directory when initializing location
 	InitLocFailedOnWorkDirMsg = `could not get working directory`

internal/archive/fsops.go

@@ -376,6 +376,18 @@ func CopyDir(src string, dst string, withParents bool, copyDirEntries func(entri
 	return copyDirEntries(entries, src, dst)
 }
 
+// FindPath returns the path or its first match in case it's a pattern. If the path doesn't exist an error is returned.
+func FindPath(path string) (string, error) {
+	sourceEntries, err := filepath.Glob(path)
+	if err == nil && len(sourceEntries) > 0 {
+		return sourceEntries[0], nil
+	}
+	if err != nil {
+		return "", errors.Wrapf(err, wrongPathMsg, path)
+	}
+	return "", errors.Errorf(wrongPathMsg, path)
+}
+
 // CopyByPatterns - copy files/directories according to patterns
 // from source folder to target folder
 // patterns are relative to source folder
@@ -387,6 +399,12 @@ func CopyByPatterns(source, target string, patterns []string) error {
 
 	logs.Logger.Infof(copyByPatternMsg, patterns[0], source, target)
 
+	// Resolve the source pattern if necessary
+	source, err := FindPath(source)
+	if err != nil {
+		return err
+	}
+
 	infoTargetDir, err := os.Stat(target)
 	if err != nil {
 		err = CreateDirIfNotExist(target)
@@ -573,6 +591,14 @@ func CopyFile(src, dst string) (rerr error) {
 		rerr = CloseFile(in, rerr)
 	}()
 
+	err = WriteFile(in, dst)
+	if err != nil {
+		return err
+	}
+	return changeTargetMode(src, dst)
+}
+
+func WriteFile(in io.Reader, dst string) (rerr error) {
 	out, err := os.Create(dst)
 	if err != nil {
 		return err
@@ -582,13 +608,7 @@ func CopyFile(src, dst string) (rerr error) {
 	}()
 
 	_, err = io.Copy(out, in)
-	if err != nil {
-		return err
-	}
-	err = changeTargetMode(src, dst)
-
 	return err
-
 }
 
 func changeTargetMode(source, target string) error {