diff --git a/cmd/cmd.go b/cmd/cmd.go index 136546dc4..123af20b0 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -25,7 +25,7 @@ var validateCmdExclude string func init() { // Add command to the root - rootCmd.AddCommand(initCmd, buildCmd, validateCmd, cleanupCmd, provideCmd, generateCmd, moduleCmd, assembleCommand, projectCmd, mergeCmd, executeCommand) + rootCmd.AddCommand(initCmd, buildCmd, validateCmd, cleanupCmd, provideCmd, generateCmd, moduleCmd, assembleCommand, projectCmd, mergeCmd, executeCommand, copyCmd) // Build module provideCmd.AddCommand(provideModuleCmd) // generate immutable commands diff --git a/cmd/exec.go b/cmd/exec.go index 895ff6844..e617e5fb8 100644 --- a/cmd/exec.go +++ b/cmd/exec.go @@ -3,11 +3,15 @@ package commands import ( "github.com/spf13/cobra" + "github.com/SAP/cloud-mta-build-tool/internal/archive" "github.com/SAP/cloud-mta-build-tool/internal/exec" ) var executeCmdCommands []string var executeCmdTimeout string +var copyCmdSrc string +var copyCmdTrg string +var copyCmdPatterns []string // Execute commands in the current working directory with a timeout. // This is used in verbose make files for implementing a timeout on module builds. @@ -26,10 +30,35 @@ var executeCommand = &cobra.Command{ SilenceErrors: true, } +// Copy files matching the specified patterns from the source path to the target path. +// This is used in verbose make files for copying artifacts from a module's dependencies before building the module. +var copyCmd = &cobra.Command{ + Use: "cp", + Short: "Copy files by patterns", + Long: "Copy files by patterns", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + err := dir.CopyByPatterns(copyCmdSrc, copyCmdTrg, copyCmdPatterns) + logError(err) + return err + }, + SilenceUsage: true, + Hidden: true, + SilenceErrors: true, +} + func init() { // set flag of execute command executeCommand.Flags().StringArrayVarP(&executeCmdCommands, "commands", "c", nil, "commands to run") executeCommand.Flags().StringVarP(&executeCmdTimeout, "timeout", "t", "", "the timeout after which the run stops, in the format [123h][123m][123s]; 10m is set as the default") + + // set flags of copy command + copyCmd.Flags().StringVarP(©CmdSrc, "source", "s", "", + "the path to the source folder") + copyCmd.Flags().StringVarP(©CmdTrg, "target", "t", "", + "the path to the target folder") + copyCmd.Flags().StringArrayVarP(©CmdPatterns, + "patterns", "p", nil, "patterns for matching the files and folders to copy") } diff --git a/cmd/exec_test.go b/cmd/exec_test.go new file mode 100644 index 000000000..73434a8e0 --- /dev/null +++ b/cmd/exec_test.go @@ -0,0 +1,93 @@ +package commands + +import ( + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/SAP/cloud-mta-build-tool/internal/archive" +) + +var _ = Describe("cp command", func() { + BeforeEach(func() { + copyCmdSrc = getTestPath("mtahtml5") + copyCmdTrg = getTestPath("result") + copyCmdPatterns = []string{} + err := dir.CreateDirIfNotExist(copyCmdTrg) + Ω(err).Should(Succeed()) + }) + + AfterEach(func() { + Ω(os.RemoveAll(getTestPath("result"))).Should(Succeed()) + }) + + It("copy - Sanity", func() { + copyCmdPatterns = []string{"mta.*", "ui5app2"} + Ω(copyCmd.RunE(nil, []string{})).Should(Succeed()) + validateFilesInDir(getTestPath("result"), []string{"mta.sh", "mta.yaml", "ui5app2/", "ui5app2/test.txt"}) + }) + It("copy should not fail when the pattern is valid but doesn't match any files", func() { + copyCmdPatterns = []string{"xxx.xxx"} + Ω(copyCmd.RunE(nil, []string{})).Should(Succeed()) + validateFilesInDir(getTestPath("result"), []string{}) + }) + It("copy should not fail when there are no patterns", func() { + copyCmdPatterns = []string{} + Ω(copyCmd.RunE(nil, []string{})).Should(Succeed()) + validateFilesInDir(getTestPath("result"), []string{}) + }) + It("copy should return error when the source folder doesn't exist", func() { + copyCmdSrc = getTestPath("mtahtml6") + copyCmdPatterns = []string{"*"} + Ω(copyCmd.RunE(nil, []string{})).Should(HaveOccurred()) + }) + It("copy should return error when a pattern is invalid", func() { + copyCmdPatterns = []string{"["} + Ω(copyCmd.RunE(nil, []string{})).Should(HaveOccurred()) + }) +}) + +// Check the folder exists and includes exactly the expected files +func validateFilesInDir(src string, expectedFilesInDir []string) { + // List all files in the folder recursively + filesInDir := make([]string, 0) + err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // Don't include the folder itself + if filepath.Clean(path) == filepath.Clean(src) { + return nil + } + relPath, err := filepath.Rel(src, path) + if err != nil { + return err + } + relPath = filepath.ToSlash(relPath) + if info.IsDir() { + relPath += "/" + } + filesInDir = append(filesInDir, relPath) + return nil + }) + Ω(err).Should(Succeed()) + + for _, expectedFile := range expectedFilesInDir { + Ω(contains(expectedFile, filesInDir)).Should(BeTrue(), fmt.Sprintf("expected %s to be in the directory; directory contains %v", expectedFile, filesInDir)) + } + for _, existingFile := range filesInDir { + Ω(contains(existingFile, expectedFilesInDir)).Should(BeTrue(), fmt.Sprintf("did not expect %s to be in the directory; directory contains %v", existingFile, filesInDir)) + } +} + +func contains(element string, elements []string) bool { + for _, el := range elements { + if el == element { + return true + } + } + return false +} diff --git a/integration/cloud_mta_build_tool_test.go b/integration/cloud_mta_build_tool_test.go index 16301cdf0..a97f9f8da 100644 --- a/integration/cloud_mta_build_tool_test.go +++ b/integration/cloud_mta_build_tool_test.go @@ -20,6 +20,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + dir "github.com/SAP/cloud-mta-build-tool/internal/archive" "github.com/SAP/cloud-mta/mta" ) @@ -374,6 +375,36 @@ modules: Ω(filepath.Join(dir, "testdata", "mta_demo", "mta_archives", demoArchiveName)).Should(BeAnExistingFile()) }) + It("Generate Verbose Makefile with module dependencies", func() { + dir, _ := os.Getwd() + path := filepath.Join(dir, "testdata", "moduledep") + archivePath := filepath.Join(path, "mta_archives", "f1_0.0.1.mtar") + tempZipPath := filepath.Join(path, "mta_archives", "data.zip") + + Ω(os.RemoveAll(filepath.Join(path, "Makefile.mta"))).Should(Succeed()) + Ω(os.RemoveAll(archivePath)).Should(Succeed()) + Ω(os.RemoveAll(filepath.Join(path, "public", "client"))).Should(Succeed()) + Ω(os.RemoveAll(tempZipPath)).Should(Succeed()) + + bin := filepath.FromSlash(binPath) + cmdOut, errString, _ := execute(bin, "init -m=verbose", path) + Ω(errString).Should(Equal("")) + Ω(cmdOut).ShouldNot(BeNil()) + // Check the MakeFile was generated + Ω(filepath.Join(path, "Makefile.mta")).Should(BeAnExistingFile()) + + // Generate mtar + bin = filepath.FromSlash("make") + execute(bin, "-f Makefile.mta p=cf", path) + // Check the archive was generated + Ω(archivePath).Should(BeAnExistingFile()) + validateMtaArchiveContents([]string{"module_with_dep/", "module_with_dep/data.zip"}, archivePath) + + // Extract data.zip and check its content + err := extractFileFromZip(archivePath, "module_with_dep/data.zip", tempZipPath) + Ω(err).Should(Succeed()) + validateArchiveContents([]string{"package.json", "client/", "client/client_package.json"}, tempZipPath) + }) }) var _ = Describe("MBT gen commands", func() { @@ -586,13 +617,48 @@ func getFileContentFromZip(path string, filename string) ([]byte, error) { return nil, fmt.Errorf(`file "%s" not found`, filename) } +func extractFileFromZip(archivePath string, filename string, dst string) (error) { + zipFile, err := zip.OpenReader(archivePath) + if err != nil { + return err + } + defer func() { + _ = zipFile.Close() + }() + var fileToExtract *zip.File = nil + for _, file := range zipFile.File { + if strings.Contains(file.Name, filename) { + fileToExtract = file + } + } + if fileToExtract == nil { + return fmt.Errorf(`file "%s" not found`, filename) + } + + in, err := fileToExtract.Open() + if err != nil { + return err + } + defer func() { + _ = in.Close() + }() + err = dir.WriteFile(in, dst) + return err +} + func validateMtaArchiveContents(expectedAdditionalFilesInArchive []string, archiveLocation string) { expectedFilesInArchive := append(expectedAdditionalFilesInArchive, "META-INF/", "META-INF/MANIFEST.MF", "META-INF/mtad.yaml") - archiveReader, err := zip.OpenReader(archiveLocation) + validateArchiveContents(expectedFilesInArchive, archiveLocation) +} + +func validateArchiveContents(expectedFilesInArchive []string, archiveLocation string) { + archive, err := zip.OpenReader(archiveLocation) Ω(err).Should(Succeed()) - defer archiveReader.Close() + defer func() { + _ = archive.Close() + }() var filesInArchive []string - for _, file := range archiveReader.File { + for _, file := range archive.File { filesInArchive = append(filesInArchive, file.Name) } for _, expectedFile := range expectedFilesInArchive { diff --git a/integration/testdata/moduledep/client/client_package.json b/integration/testdata/moduledep/client/client_package.json new file mode 100644 index 000000000..d00a2b3d5 --- /dev/null +++ b/integration/testdata/moduledep/client/client_package.json @@ -0,0 +1,34 @@ +{ + "name": "f1.client", + "title": "f1", + "version": "0.0.1", + "description": "", + "scripts": { + "postinstall": "node ./scripts/bundle.js && npm run lint", + "lint": "node ./node_modules/eslint/bin/eslint . --ext=.js,.json" + }, + "devDependencies": { + "eslint": "4.12.1", + "@sap/eslint-plugin-webide-feature": "1.3.15", + "@sap-webide/webide-client-tools": "latest" + }, + "bundledPlugins": { + "p1": "file:src/p1" + }, + "files": [ + "i18n", + "src", + "config-preload.js", + "config-preload.json", + "package.json" + ], + "webidePreloads": { + "js": [ + "config-preload.js", + "i18n/config-preload.js" + ], + "config": [ + "config-preload.json" + ] + } +} diff --git a/integration/testdata/moduledep/mta.yaml b/integration/testdata/moduledep/mta.yaml new file mode 100644 index 000000000..41ace10a9 --- /dev/null +++ b/integration/testdata/moduledep/mta.yaml @@ -0,0 +1,21 @@ +_schema-version: "2.0.0" +ID: f1 +version: 0.0.1 + +modules: + - name: module_with_dep + type: html5 + path: public + build-parameters: + builder: zip + requires: + - name: m2 + artifacts: ["*"] + target-path: "client" + - name: m2 + type: html5 + path: client + build-parameters: + builder: zip + supported-platforms: [] + diff --git a/integration/testdata/moduledep/public/package.json b/integration/testdata/moduledep/public/package.json new file mode 100644 index 000000000..0638a8322 --- /dev/null +++ b/integration/testdata/moduledep/public/package.json @@ -0,0 +1,10 @@ +{ + "name": "f1.approuter", + "version": "0.0.1", + "scripts": { + "start": "node node_modules/@sap/approuter/approuter.js" + }, + "dependencies": { + "@sap/approuter": "2.7.0" + } +} diff --git a/internal/archive/archive_msg.go b/internal/archive/archive_msg.go index 4b39838d6..fe752e88e 100644 --- a/internal/archive/archive_msg.go +++ b/internal/archive/archive_msg.go @@ -11,6 +11,7 @@ const ( copyByPatternFailedOnCreateMsg = `could not copy files matching the patterns [%s,...] from the "%s" folder to the "%s" folder: could not create the "%s" folder` copyByPatternFailedOnTargetMsg = `could not copy files matching the patterns [%s,...] from the "%s" folder to the "%s" folder: "%s" is not a folder` copyByPatternFailedOnMatchMsg = `could not copy files matching the "%s" pattern from the "%s" folder to the "%s": could not get list of files matching the "%s" pattern` + wrongPathMsg = `could not find the "%s" path` // InitLocFailedOnWorkDirMsg - message raised on getting working directory when initializing location InitLocFailedOnWorkDirMsg = `could not get working directory` diff --git a/internal/archive/fsops.go b/internal/archive/fsops.go index c5d3108d4..2c6e1c3f4 100755 --- a/internal/archive/fsops.go +++ b/internal/archive/fsops.go @@ -376,6 +376,18 @@ func CopyDir(src string, dst string, withParents bool, copyDirEntries func(entri return copyDirEntries(entries, src, dst) } +// FindPath returns the path or its first match in case it's a pattern. If the path doesn't exist an error is returned. +func FindPath(path string) (string, error) { + sourceEntries, err := filepath.Glob(path) + if err == nil && len(sourceEntries) > 0 { + return sourceEntries[0], nil + } + if err != nil { + return "", errors.Wrapf(err, wrongPathMsg, path) + } + return "", errors.Errorf(wrongPathMsg, path) +} + // CopyByPatterns - copy files/directories according to patterns // from source folder to target folder // patterns are relative to source folder @@ -387,6 +399,12 @@ func CopyByPatterns(source, target string, patterns []string) error { logs.Logger.Infof(copyByPatternMsg, patterns[0], source, target) + // Resolve the source pattern if necessary + source, err := FindPath(source) + if err != nil { + return err + } + infoTargetDir, err := os.Stat(target) if err != nil { err = CreateDirIfNotExist(target) @@ -573,6 +591,14 @@ func CopyFile(src, dst string) (rerr error) { rerr = CloseFile(in, rerr) }() + err = WriteFile(in, dst) + if err != nil { + return err + } + return changeTargetMode(src, dst) +} + +func WriteFile(in io.Reader, dst string) (rerr error) { out, err := os.Create(dst) if err != nil { return err @@ -582,13 +608,7 @@ func CopyFile(src, dst string) (rerr error) { }() _, err = io.Copy(out, in) - if err != nil { - return err - } - err = changeTargetMode(src, dst) - return err - } func changeTargetMode(source, target string) error { diff --git a/internal/archive/fsops_test.go b/internal/archive/fsops_test.go index 2e8b49d7d..3d1f3f83b 100644 --- a/internal/archive/fsops_test.go +++ b/internal/archive/fsops_test.go @@ -138,6 +138,47 @@ var _ = Describe("FSOPS", func() { ) }) + var _ = Describe("FindPath", func() { + It("returns file path for existing file", func() { + path := getFullPath("testdata", "findpath", "folder1", "file1.txt") + found, err := FindPath(path) + Ω(err).Should(Succeed()) + Ω(found).Should(Equal(path)) + }) + It("returns folder path for existing folder", func() { + path := getFullPath("testdata", "findpath", "folder1") + found, err := FindPath(path) + Ω(err).Should(Succeed()) + Ω(found).Should(Equal(path)) + }) + It("returns first found file for pattern", func() { + path := getFullPath("testdata", "findpath", "folder1", "*.txt") + found, err := FindPath(path) + Ω(err).Should(Succeed()) + Ω(found).Should(Equal(getFullPath("testdata", "findpath", "folder1", "file1.txt"))) + }) + It("returns first found folder for pattern", func() { + path := getFullPath("testdata", "findpath", "folder*") + found, err := FindPath(path) + Ω(err).Should(Succeed()) + Ω(found).Should(Equal(getFullPath("testdata", "findpath", "folder1"))) + }) + It("returns error when path is not found for path", func() { + path := getFullPath("testdata", "findpath", "fff") + _, err := FindPath(path) + Ω(err).Should(HaveOccurred()) + }) + It("returns error when no files or folders are found for pattern", func() { + path := getFullPath("testdata", "findpath", "fff*") + _, err := FindPath(path) + Ω(err).Should(HaveOccurred()) + }) + It("returns error when pattern is invalid", func() { + _, err := FindPath(getFullPath("testdata", "findpath", "*[")) + Ω(err).Should(HaveOccurred()) + }) + }) + var _ = Describe("utils", func() { BeforeEach(func() { fileInfoProvider = &mockFileInfoProvider{} @@ -333,7 +374,7 @@ var _ = Describe("FSOPS", func() { var _ = Describe("Copy Entries", func() { AfterEach(func() { - os.RemoveAll(getFullPath("testdata", "result")) + Ω(os.RemoveAll(getFullPath("testdata", "result"))).Should(Succeed()) }) It("Sanity", func() { @@ -352,7 +393,6 @@ var _ = Describe("FSOPS", func() { } Ω(CopyEntries(filesWrapped, sourcePath, targetPath)).Should(Succeed()) Ω(countFilesInDir(sourcePath) - 1).Should(Equal(countFilesInDir(targetPath))) - os.RemoveAll(targetPath) }) It("Sanity - copy in parallel", func() { sourcePath := getFullPath("testdata", "level2", "level3") @@ -370,7 +410,6 @@ var _ = Describe("FSOPS", func() { } Ω(CopyEntriesInParallel(filesWrapped, sourcePath, targetPath)).Should(Succeed()) Ω(countFilesInDir(sourcePath) - 1).Should(Equal(countFilesInDir(targetPath))) - os.RemoveAll(targetPath) }) }) @@ -451,6 +490,8 @@ var _ = Describe("FSOPS", func() { Entry("Wrong pattern ", getFullPath("testdata", "result"), "ui2", []string{"[a,b"}), Entry("Empty target path", "", "ui2", []string{"[a,b"}), + Entry("Source path doesn't exist", + getFullPath("testdata", "result"), "aaa", []string{"*"}), ) }) diff --git a/internal/archive/testdata/findpath/folder1/file1.txt b/internal/archive/testdata/findpath/folder1/file1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/internal/artifacts/module_arch.go b/internal/artifacts/module_arch.go index ca2d83356..18fe0120b 100644 --- a/internal/artifacts/module_arch.go +++ b/internal/artifacts/module_arch.go @@ -119,7 +119,7 @@ func packModule(source dir.IModule, target dir.ITargetPath, module *mta.Module, logs.Logger.Info(fmt.Sprintf(buildResultMsg, moduleName, source.GetTargetModuleDir(moduleName))) - sourceArtifact, _, _, err := buildops.GetModuleSourceArtifactPath(source, false, module, defaultBuildResult) + sourceArtifact, err := buildops.GetModuleSourceArtifactPath(source, false, module, defaultBuildResult, true) if err != nil { return errors.Wrapf(err, packFailedOnBuildArtifactMsg, moduleName) } diff --git a/internal/buildops/build_params.go b/internal/buildops/build_params.go index 8a43226e7..b570d124a 100644 --- a/internal/buildops/build_params.go +++ b/internal/buildops/build_params.go @@ -36,9 +36,9 @@ type BuildRequires struct { TargetPath string `yaml:"target-path,omitempty"` } -// getBuildRequires - gets Requires property of module's build-params property +// GetBuildRequires - gets Requires property of module's build-params property // as generic property and converts it to slice of BuildRequires structures -func getBuildRequires(module *mta.Module) []BuildRequires { +func GetBuildRequires(module *mta.Module) []BuildRequires { // check existence of module's build-params.require property if module.BuildParams != nil && module.BuildParams[requiresParam] != nil { requires := module.BuildParams[requiresParam].([]interface{}) @@ -87,44 +87,51 @@ func getStrParam(m map[string]interface{}, param string) string { // 1. Cyclic dependencies // 2. Dependency on not defined module -// ProcessRequirements - Processes build requirement of module (using moduleName). -func ProcessRequirements(ep dir.ISourceModule, mta *mta.MTA, requires *BuildRequires, moduleName string) error { - +// GetRequiresArtifacts returns the source path, target path and patterns of files and folders to copy from a module's requires section +func GetRequiresArtifacts(ep dir.ISourceModule, mta *mta.MTA, requires *BuildRequires, moduleName string, resolveBuildResult bool) (source string, target string, patterns []string, err error) { // validate module names - both in process and required module, err := mta.GetModuleByName(moduleName) if err != nil { - return errors.Wrapf(err, reqFailedOnModuleGetMsg, moduleName, requires.Name, moduleName) + return "", "", nil, errors.Wrapf(err, reqFailedOnModuleGetMsg, moduleName, requires.Name, moduleName) } requiredModule, err := mta.GetModuleByName(requires.Name) if err != nil { - return errors.Wrapf(err, reqFailedOnModuleGetMsg, moduleName, requires.Name, requires.Name) + return "", "", nil, errors.Wrapf(err, reqFailedOnModuleGetMsg, moduleName, requires.Name, requires.Name) } _, defaultBuildResult, err := commands.CommandProvider(*requiredModule) if err != nil { - return errors.Wrapf(err, reqFailedOnCommandsGetMsg, moduleName, requires.Name, requires.Name) + return "", "", nil, errors.Wrapf(err, reqFailedOnCommandsGetMsg, moduleName, requires.Name, requires.Name) } // Build paths for artifacts copying - sourcePath, _, _, err := GetModuleSourceArtifactPath(ep, false, requiredModule, defaultBuildResult) + sourcePath, err := GetModuleSourceArtifactPath(ep, false, requiredModule, defaultBuildResult, resolveBuildResult) if err != nil { - return errors.Wrapf(err, reqFailedOnBuildResultMsg, moduleName, requires.Name) + return "", "", nil, errors.Wrapf(err, reqFailedOnBuildResultMsg, moduleName, requires.Name) } targetPath := getRequiredTargetPath(ep, module, requires) + return sourcePath, targetPath, requires.Artifacts, nil +} +// ProcessRequirements - Processes build requirement of module (using moduleName). +func ProcessRequirements(ep dir.ISourceModule, mta *mta.MTA, requires *BuildRequires, moduleName string) error { + sourcePath, targetPath, artifacts, err := GetRequiresArtifacts(ep, mta, requires, moduleName, true) + if err != nil { + return err + } // execute copy of artifacts - err = dir.CopyByPatterns(sourcePath, targetPath, requires.Artifacts) + err = dir.CopyByPatterns(sourcePath, targetPath, artifacts) if err != nil { - return errors.Wrapf(err, reqFailedOnCopyMsg, moduleName, requiredModule.Name) + return errors.Wrapf(err, reqFailedOnCopyMsg, moduleName, requires.Name) } return nil } // GetModuleSourceArtifactPath - get the module's artifact that has to be archived in the mtar, from the project sources -func GetModuleSourceArtifactPath(loc dir.ISourceModule, depDesc bool, module *mta.Module, defaultBuildResult string) (path string, isFolder, isArchive bool, e error) { +func GetModuleSourceArtifactPath(loc dir.ISourceModule, depDesc bool, module *mta.Module, defaultBuildResult string, resolveBuildResult bool) (path string, e error) { if module.Path == "" { - return "", false, false, nil + return "", nil } path = loc.GetSourceModuleDir(module.Path) if !depDesc { @@ -133,18 +140,20 @@ func GetModuleSourceArtifactPath(loc dir.ISourceModule, depDesc bool, module *mt if module.BuildParams != nil && module.BuildParams[buildResultParam] != nil { buildResult, ok = module.BuildParams[buildResultParam].(string) if !ok { - return "", false, false, errors.Errorf(WrongBuildResultMsg, module.BuildParams[buildResultParam], module.Name) + return "", errors.Errorf(WrongBuildResultMsg, module.BuildParams[buildResultParam], module.Name) } } if buildResult != "" { - path = findPath(filepath.Join(path, buildResult)) + path = filepath.Join(path, buildResult) + if resolveBuildResult { + path, e = dir.FindPath(path) + if e != nil { + return "", e + } + } } } - isArchive, isFolder, err := IsArchive(path) - if err != nil { - return "", false, false, errors.Wrapf(err, wrongPathMsg, path) - } - return path, isArchive, isFolder, nil + return path, nil } // IsArchive - check if file is a folder or an archive @@ -163,15 +172,6 @@ func IsArchive(path string) (isArchive, isFolder bool, e error) { return isArchive, isFolder, nil } -func findPath(pathOrPattern string) string { - path := pathOrPattern - sourceEntries, err := filepath.Glob(path) - if err == nil && len(sourceEntries) > 0 { - path = sourceEntries[0] - } - return path -} - // GetModuleTargetArtifactPath - get the path to where the module's artifact should be created in the temp folder, from which it's archived in the mtar func GetModuleTargetArtifactPath(source dir.ISourceModule, loc dir.ITargetPath, depDesc bool, module *mta.Module, defaultBuildResult string) (path string, toArchive bool, e error) { if module.Path == "" { @@ -180,10 +180,14 @@ func GetModuleTargetArtifactPath(source dir.ISourceModule, loc dir.ITargetPath, if depDesc { path = filepath.Join(loc.GetTargetTmpDir(), module.Path) } else { - moduleSourceArtifactPath, isArchive, isFolder, err := GetModuleSourceArtifactPath(source, depDesc, module, defaultBuildResult) + moduleSourceArtifactPath, err := GetModuleSourceArtifactPath(source, depDesc, module, defaultBuildResult, true) if err != nil { return "", false, err } + isArchive, isFolder, err := IsArchive(moduleSourceArtifactPath) + if err != nil { + return "", false, errors.Wrapf(err, wrongPathMsg, moduleSourceArtifactPath) + } artifactName, artifactExt, err := getArtifactInfo(isArchive, module, moduleSourceArtifactPath) if err != nil { return "", false, err diff --git a/internal/buildops/build_params_test.go b/internal/buildops/build_params_test.go index d4911836c..d50a262f3 100644 --- a/internal/buildops/build_params_test.go +++ b/internal/buildops/build_params_test.go @@ -18,7 +18,7 @@ var _ = Describe("BuildParams", func() { var _ = DescribeTable("valid cases", func(module *mta.Module, expected string) { loc := &dir.Loc{SourcePath: getTestPath("mtahtml5")} - path, _, _, err := GetModuleSourceArtifactPath(loc, false, module, "") + path, err := GetModuleSourceArtifactPath(loc, false, module, "", true) Ω(err).Should(Succeed()) Ω(path).Should(Equal(expected)) }, @@ -32,8 +32,8 @@ var _ = Describe("BuildParams", func() { var _ = Describe("GetBuildResultsPath", func() { It("empty path, no build results", func() { module := &mta.Module{} - buildResult, _, _, _ := GetModuleSourceArtifactPath( - &dir.Loc{SourcePath: getTestPath("testbuildparams", "ui2", "deep", "folder")}, false, module, "") + buildResult, _ := GetModuleSourceArtifactPath( + &dir.Loc{SourcePath: getTestPath("testbuildparams", "ui2", "deep", "folder")}, false, module, "", true) Ω(buildResult).Should(Equal("")) }) @@ -42,8 +42,8 @@ var _ = Describe("BuildParams", func() { Path: "inui2", BuildParams: map[string]interface{}{buildResultParam: "*.txt"}, } - buildResult, _, _, _ := GetModuleSourceArtifactPath( - &dir.Loc{SourcePath: getTestPath("testbuildparams", "ui2", "deep", "folder")}, false, module, "") + buildResult, _ := GetModuleSourceArtifactPath( + &dir.Loc{SourcePath: getTestPath("testbuildparams", "ui2", "deep", "folder")}, false, module, "", true) Ω(buildResult).Should(HaveSuffix("anotherfile.txt")) }) @@ -51,16 +51,16 @@ var _ = Describe("BuildParams", func() { module := &mta.Module{ Path: "inui2", } - buildResult, _, _, _ := GetModuleSourceArtifactPath( - &dir.Loc{SourcePath: getTestPath("testbuildparams", "ui2", "deep", "folder")}, false, module, "*.txt") + buildResult, _ := GetModuleSourceArtifactPath( + &dir.Loc{SourcePath: getTestPath("testbuildparams", "ui2", "deep", "folder")}, false, module, "*.txt", true) Ω(buildResult).Should(HaveSuffix("anotherfile.txt")) }) It("default build results - no file answers pattern", func() { module := &mta.Module{ Path: "inui2", } - _, _, _, err := GetModuleSourceArtifactPath( - &dir.Loc{SourcePath: getTestPath("testbuildparams", "ui2", "deep", "folder")}, false, module, "b*.txt") + _, err := GetModuleSourceArtifactPath( + &dir.Loc{SourcePath: getTestPath("testbuildparams", "ui2", "deep", "folder")}, false, module, "b*.txt", true) Ω(err).Should(HaveOccurred()) }) }) @@ -170,6 +170,11 @@ var _ = Describe("GetModuleTargetArtifactPath", func() { Ω(err).Should(Succeed()) Ω(path).Should(BeEmpty()) }) + It("fails when path doesn't exist", func() { + loc := dir.Loc{SourcePath: getTestPath("mtahtml5"), TargetPath: getTestPath("result")} + _, _, err := GetModuleTargetArtifactPath(&loc, &loc, false, &mta.Module{Path: "abc"}, "") + Ω(err).Should(HaveOccurred()) + }) It("deployment descriptor", func() { loc := dir.Loc{SourcePath: getTestPath("mtahtml5"), TargetPath: getTestPath("result")} path, _, err := GetModuleTargetArtifactPath(&loc, &loc, true, &mta.Module{Path: "abc"}, "") @@ -275,7 +280,7 @@ var _ = Describe("Process complex list of requirements", func() { mtaObj, _ := lp.ParseFile() for _, m := range mtaObj.Modules { if m.Name == "node" { - for _, r := range getBuildRequires(m) { + for _, r := range GetBuildRequires(m) { err := ProcessRequirements(&lp, mtaObj, &r, "node") if err != nil { fmt.Println("error occurred during build process requirements ") diff --git a/internal/buildops/modules_deps.go b/internal/buildops/modules_deps.go index 0abaac838..08d5eca6f 100644 --- a/internal/buildops/modules_deps.go +++ b/internal/buildops/modules_deps.go @@ -47,7 +47,7 @@ func ProcessDependencies(mtaParser dir.IMtaParser, moduleSource dir.ISourceModul if err != nil { return err } - requires := getBuildRequires(module) + requires := GetBuildRequires(module) if len(requires) > 0 { for _, req := range requires { e := ProcessRequirements(moduleSource, m, &req, module.Name) @@ -77,7 +77,7 @@ func getModulesOrder(m *mta.MTA) ([]string, error) { var graph = make(graphs) for index, module := range m.Modules { deps := mapset.NewSet() - requires := getBuildRequires(module) + requires := GetBuildRequires(module) if len(requires) > 0 { for _, req := range requires { _, err := m.GetModuleByName(req.Name) diff --git a/internal/tpl/make_verbose.go b/internal/tpl/make_verbose.go index d64b2eb59..c3c93bc89 100644 --- a/internal/tpl/make_verbose.go +++ b/internal/tpl/make_verbose.go @@ -1,4 +1,4 @@ package tpl // makeVerbose - do not edit -var makeVerbose = []byte{0x23, 0x20, 0x4c, 0x69, 0x73, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0xa, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x7b, 0x7b, 0x2d, 0x20, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x20, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x7d, 0x7d, 0x20, 0x7b, 0x7b, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x7d, 0x7b, 0x7b, 0x65, 0x6e, 0x64, 0x7d, 0x7d, 0xa, 0xa, 0x23, 0x20, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x20, 0x61, 0x6c, 0x6c, 0x20, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x20, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x73, 0xa, 0x7b, 0x7b, 0x2d, 0x20, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x20, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x7d, 0x7d, 0xa, 0x23, 0x20, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x20, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x20, 0x7b, 0x7b, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x7d, 0xa, 0x7b, 0x7b, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x7d, 0x3a, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0xa, 0x7b, 0x7b, 0x22, 0x5c, 0x74, 0x22, 0x7d, 0x7d, 0x40, 0x63, 0x64, 0x20, 0x22, 0x24, 0x28, 0x50, 0x52, 0x4f, 0x4a, 0x5f, 0x44, 0x49, 0x52, 0x29, 0x2f, 0x7b, 0x7b, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x7d, 0x7d, 0x22, 0x20, 0x26, 0x26, 0x20, 0x24, 0x28, 0x4d, 0x42, 0x54, 0x29, 0x20, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x20, 0x7b, 0x7b, 0x2d, 0x20, 0x69, 0x66, 0x20, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x7d, 0x7d, 0x20, 0x2d, 0x74, 0x3d, 0x7b, 0x7b, 0x24, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x54, 0x6f, 0x53, 0x68, 0x65, 0x6c, 0x6c, 0x41, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x20, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x7d, 0x7d, 0x7b, 0x7b, 0x65, 0x6e, 0x64, 0x7d, 0x7d, 0x20, 0x7b, 0x7b, 0x2d, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x24, 0x63, 0x6d, 0x64, 0x73, 0x20, 0x3a, 0x3d, 0x20, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x20, 0x2e, 0x7d, 0x7d, 0x7b, 0x7b, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x20, 0x24, 0x69, 0x2c, 0x20, 0x24, 0x63, 0x6d, 0x64, 0x3a, 0x3d, 0x24, 0x63, 0x6d, 0x64, 0x73, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x7d, 0x7d, 0x20, 0x2d, 0x63, 0x3d, 0x7b, 0x7b, 0x24, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x54, 0x6f, 0x53, 0x68, 0x65, 0x6c, 0x6c, 0x41, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x20, 0x2e, 0x7d, 0x7d, 0x7b, 0x7b, 0x65, 0x6e, 0x64, 0x7d, 0x7d, 0x7b, 0x7b, 0x65, 0x6e, 0x64, 0x7d, 0x7d, 0xa, 0x23, 0x20, 0x50, 0x61, 0x63, 0x6b, 0x20, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x20, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0xa, 0x7b, 0x7b, 0x22, 0x5c, 0x74, 0x22, 0x7d, 0x7d, 0x40, 0x24, 0x28, 0x4d, 0x42, 0x54, 0x29, 0x20, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x20, 0x70, 0x61, 0x63, 0x6b, 0x20, 0x2d, 0x6d, 0x3d, 0x7b, 0x7b, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x7d, 0x20, 0x2d, 0x70, 0x3d, 0x24, 0x7b, 0x70, 0x7d, 0x20, 0x2d, 0x74, 0x3d, 0x24, 0x7b, 0x74, 0x7d, 0x20, 0x7b, 0x7b, 0x2d, 0x20, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x41, 0x72, 0x67, 0x20, 0x22, 0x2d, 0x65, 0x22, 0x7d, 0x7d, 0xa, 0x7b, 0x7b, 0x65, 0x6e, 0x64, 0x7d, 0x7d, 0xa} +var makeVerbose = []byte{0x23, 0x20, 0x4c, 0x69, 0x73, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0xa, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x7b, 0x7b, 0x2d, 0x20, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x20, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x7d, 0x7d, 0x20, 0x7b, 0x7b, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x7d, 0x7b, 0x7b, 0x65, 0x6e, 0x64, 0x7d, 0x7d, 0xa, 0xa, 0x23, 0x20, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x20, 0x61, 0x6c, 0x6c, 0x20, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x20, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x73, 0xa, 0x7b, 0x7b, 0x2d, 0x20, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x20, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x7d, 0x7d, 0xa, 0x23, 0x20, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x20, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x20, 0x7b, 0x7b, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x7d, 0xa, 0x7b, 0x7b, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x7d, 0x3a, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x20, 0x7b, 0x7b, 0x2d, 0x20, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x20, 0x24, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x44, 0x65, 0x70, 0x73, 0x20, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x7d, 0x20, 0x7b, 0x7b, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x7d, 0x7b, 0x7b, 0x65, 0x6e, 0x64, 0x7d, 0x7d, 0xa, 0x7b, 0x7b, 0x2d, 0x20, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x20, 0x24, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x44, 0x65, 0x70, 0x73, 0x20, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x7d, 0x7b, 0x7b, 0x22, 0x5c, 0x6e, 0x5c, 0x74, 0x22, 0x7d, 0x7d, 0x40, 0x24, 0x28, 0x4d, 0x42, 0x54, 0x29, 0x20, 0x63, 0x70, 0x20, 0x2d, 0x73, 0x3d, 0x7b, 0x7b, 0x24, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x54, 0x6f, 0x53, 0x68, 0x65, 0x6c, 0x6c, 0x41, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x20, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x61, 0x74, 0x68, 0x7d, 0x7d, 0x20, 0x2d, 0x74, 0x3d, 0x7b, 0x7b, 0x24, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x54, 0x6f, 0x53, 0x68, 0x65, 0x6c, 0x6c, 0x41, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x20, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x7d, 0x7d, 0x20, 0x7b, 0x7b, 0x2d, 0x20, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x20, 0x2e, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x7d, 0x7d, 0x20, 0x2d, 0x70, 0x3d, 0x7b, 0x7b, 0x24, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x54, 0x6f, 0x53, 0x68, 0x65, 0x6c, 0x6c, 0x41, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x20, 0x2e, 0x7d, 0x7d, 0x7b, 0x7b, 0x65, 0x6e, 0x64, 0x7d, 0x7d, 0x7b, 0x7b, 0x65, 0x6e, 0x64, 0x7d, 0x7d, 0xa, 0x7b, 0x7b, 0x22, 0x5c, 0x74, 0x22, 0x7d, 0x7d, 0x40, 0x63, 0x64, 0x20, 0x22, 0x24, 0x28, 0x50, 0x52, 0x4f, 0x4a, 0x5f, 0x44, 0x49, 0x52, 0x29, 0x2f, 0x7b, 0x7b, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x7d, 0x7d, 0x22, 0x20, 0x26, 0x26, 0x20, 0x24, 0x28, 0x4d, 0x42, 0x54, 0x29, 0x20, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x20, 0x7b, 0x7b, 0x2d, 0x20, 0x69, 0x66, 0x20, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x7d, 0x7d, 0x20, 0x2d, 0x74, 0x3d, 0x7b, 0x7b, 0x24, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x54, 0x6f, 0x53, 0x68, 0x65, 0x6c, 0x6c, 0x41, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x20, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x2e, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x7d, 0x7d, 0x7b, 0x7b, 0x65, 0x6e, 0x64, 0x7d, 0x7d, 0x20, 0x7b, 0x7b, 0x2d, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x24, 0x63, 0x6d, 0x64, 0x73, 0x20, 0x3a, 0x3d, 0x20, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x20, 0x2e, 0x7d, 0x7d, 0x7b, 0x7b, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x20, 0x24, 0x69, 0x2c, 0x20, 0x24, 0x63, 0x6d, 0x64, 0x3a, 0x3d, 0x24, 0x63, 0x6d, 0x64, 0x73, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x7d, 0x7d, 0x20, 0x2d, 0x63, 0x3d, 0x7b, 0x7b, 0x24, 0x2e, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x54, 0x6f, 0x53, 0x68, 0x65, 0x6c, 0x6c, 0x41, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x20, 0x2e, 0x7d, 0x7d, 0x7b, 0x7b, 0x65, 0x6e, 0x64, 0x7d, 0x7d, 0x7b, 0x7b, 0x65, 0x6e, 0x64, 0x7d, 0x7d, 0xa, 0x23, 0x20, 0x50, 0x61, 0x63, 0x6b, 0x20, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x20, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x20, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0xa, 0x7b, 0x7b, 0x22, 0x5c, 0x74, 0x22, 0x7d, 0x7d, 0x40, 0x24, 0x28, 0x4d, 0x42, 0x54, 0x29, 0x20, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x20, 0x70, 0x61, 0x63, 0x6b, 0x20, 0x2d, 0x6d, 0x3d, 0x7b, 0x7b, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x7d, 0x7d, 0x20, 0x2d, 0x70, 0x3d, 0x24, 0x7b, 0x70, 0x7d, 0x20, 0x2d, 0x74, 0x3d, 0x24, 0x7b, 0x74, 0x7d, 0x20, 0x7b, 0x7b, 0x2d, 0x20, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x41, 0x72, 0x67, 0x20, 0x22, 0x2d, 0x65, 0x22, 0x7d, 0x7d, 0xa, 0x7b, 0x7b, 0x65, 0x6e, 0x64, 0x7d, 0x7d, 0xa} diff --git a/internal/tpl/make_verbose.txt b/internal/tpl/make_verbose.txt index 7a9b5158c..4171e9539 100644 --- a/internal/tpl/make_verbose.txt +++ b/internal/tpl/make_verbose.txt @@ -4,7 +4,8 @@ modules = {{- range .File.Modules}} {{.Name}}{{end}} # Execute all modules builds {{- range .File.Modules}} # build module {{.Name}} -{{.Name}}: validate +{{.Name}}: validate {{- range $.GetModuleDeps .Name}} {{.Name}}{{end}} +{{- range $.GetModuleDeps .Name}}{{"\n\t"}}@$(MBT) cp -s={{$.ConvertToShellArgument .SourcePath}} -t={{$.ConvertToShellArgument .TargetPath}} {{- range .Patterns}} -p={{$.ConvertToShellArgument .}}{{end}}{{end}} {{"\t"}}@cd "$(PROJ_DIR)/{{.Path}}" && $(MBT) execute {{- if .BuildParams.timeout}} -t={{$.ConvertToShellArgument .BuildParams.timeout}}{{end}} {{- with $cmds := CommandProvider .}}{{range $i, $cmd:=$cmds.Command}} -c={{$.ConvertToShellArgument .}}{{end}}{{end}} # Pack module build artifacts {{"\t"}}@$(MBT) module pack -m={{.Name}} -p=${p} -t=${t} {{- ExtensionsArg "-e"}} diff --git a/internal/tpl/makefile.go b/internal/tpl/makefile.go index 6880e2030..1007b6066 100644 --- a/internal/tpl/makefile.go +++ b/internal/tpl/makefile.go @@ -11,6 +11,7 @@ import ( "github.com/pkg/errors" "github.com/SAP/cloud-mta-build-tool/internal/archive" + "github.com/SAP/cloud-mta-build-tool/internal/buildops" "github.com/SAP/cloud-mta-build-tool/internal/commands" "github.com/SAP/cloud-mta-build-tool/internal/logs" "github.com/SAP/cloud-mta-build-tool/internal/proc" @@ -33,7 +34,7 @@ func ExecuteMake(source, target string, extensions []string, name, mode string, if err != nil { return errors.Wrapf(err, genFailedOnInitLocMsg, name) } - err = genMakefile(loc, loc, loc, loc.GetExtensionFilePaths(), name, mode, useDefaultMbt) + err = genMakefile(loc, loc, loc, loc, loc.GetExtensionFilePaths(), name, mode, useDefaultMbt) if err != nil { return err } @@ -42,20 +43,28 @@ func ExecuteMake(source, target string, extensions []string, name, mode string, } // genMakefile - Generate the makefile -func genMakefile(mtaParser dir.IMtaParser, loc dir.ITargetPath, desc dir.IDescriptor, extensionFilePaths []string, makeFilename, mode string, useDefaultMbt bool) error { +func genMakefile(mtaParser dir.IMtaParser, loc dir.ITargetPath, srcLoc dir.ISourceModule, desc dir.IDescriptor, extensionFilePaths []string, makeFilename, mode string, useDefaultMbt bool) error { tpl, err := getTplCfg(mode, desc.IsDeploymentDescriptor()) if err != nil { return err } if err == nil { tpl.depDesc = desc.GetDescriptor() - err = makeFile(mtaParser, loc, extensionFilePaths, makeFilename, &tpl, useDefaultMbt) + err = makeFile(mtaParser, loc, srcLoc, extensionFilePaths, makeFilename, &tpl, useDefaultMbt) } return err } type templateData struct { File mta.MTA + Loc dir.ISourceModule +} + +type templateDepData struct { + Name string + SourcePath string + TargetPath string + Patterns []string } // ConvertToShellArgument wraps a string in quotation marks if necessary and escapes necessary characters in it, @@ -64,8 +73,25 @@ func (data templateData) ConvertToShellArgument(s string) string { return shellquote.Join(s) } +func (data templateData) GetModuleDeps(moduleName string) ([]templateDepData, error) { + module, e := data.File.GetModuleByName(moduleName) + if e != nil { + return nil, e + } + requires := buildops.GetBuildRequires(module) + templateDeps := make([]templateDepData, len(requires)) + for index, req := range requires { + sourcePath, targetPath, artifacts, e := buildops.GetRequiresArtifacts(data.Loc, &data.File, &req, moduleName, false) + if e != nil { + return nil, e + } + templateDeps[index] = templateDepData{req.Name, sourcePath, targetPath, artifacts} + } + return templateDeps, nil +} + // makeFile - generate makefile form templates -func makeFile(mtaParser dir.IMtaParser, loc dir.ITargetPath, extensionFilePaths []string, makeFilename string, tpl *tplCfg, useDefaultMbt bool) (e error) { +func makeFile(mtaParser dir.IMtaParser, loc dir.ITargetPath, srcLoc dir.ISourceModule, extensionFilePaths []string, makeFilename string, tpl *tplCfg, useDefaultMbt bool) (e error) { // template data data := templateData{} @@ -81,8 +107,16 @@ func makeFile(mtaParser dir.IMtaParser, loc dir.ITargetPath, extensionFilePaths return errors.Wrapf(err, genFailedMsg, makeFilename) } + // Check for circular build dependencies between the modules. The error message from make is not clear so we + // should give an error here during the generation of the makefile. + _, e = buildops.GetModulesNames(m) + if e != nil { + return e + } + // Template data data.File = *m + data.Loc = srcLoc // path for creating the file target := loc.GetTarget() diff --git a/internal/tpl/makefile_test.go b/internal/tpl/makefile_test.go index 97055ef98..88917d324 100644 --- a/internal/tpl/makefile_test.go +++ b/internal/tpl/makefile_test.go @@ -8,6 +8,7 @@ import ( "runtime" "strings" + "github.com/kballard/go-shellquote" . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" @@ -16,6 +17,7 @@ import ( "github.com/SAP/cloud-mta-build-tool/internal/archive" "github.com/SAP/cloud-mta-build-tool/internal/logs" "github.com/SAP/cloud-mta-build-tool/internal/version" + "github.com/SAP/cloud-mta/mta" ) const ( @@ -37,6 +39,10 @@ func getMakeFileContent(filePath string) string { return removeSpecialSymbols(expected) } +func escapePath(parts ...string) string { + return shellquote.Join(filepath.Join(parts...)) +} + var _ = Describe("Makefile", func() { var ( @@ -102,33 +108,41 @@ makefile_version: 0.0.0 }) It("Sanity", func() { ep := dir.Loc{SourcePath: filepath.Join(wd, "testdata"), TargetPath: filepath.Join(wd, "testdata"), Descriptor: "dev"} - Ω(makeFile(&ep, &ep, nil, makeFileName, &tpl, true)).Should(Succeed()) + Ω(makeFile(&ep, &ep, &ep, nil, makeFileName, &tpl, true)).Should(Succeed()) Ω(makeFileFullPath).Should(BeAnExistingFile()) Ω(getMakeFileContent(makeFileFullPath)).Should(Equal(expectedMakeFileContent)) }) It("Create make file in folder that does not exist", func() { ep := dir.Loc{SourcePath: filepath.Join(wd, "testdata"), TargetPath: filepath.Join(wd, "testdata", "someFolder"), Descriptor: "dev"} - Ω(makeFile(&ep, &ep, nil, makeFileName, &tpl, true)).Should(Succeed()) + Ω(makeFile(&ep, &ep, &ep, nil, makeFileName, &tpl, true)).Should(Succeed()) filename := filepath.Join(ep.GetTarget(), makeFileName) Ω(filename).Should(BeAnExistingFile()) Ω(getMakeFileContent(filename)).Should(Equal(expectedMakeFileContent)) }) It("genMakefile testing with wrong mta yaml file", func() { ep := dir.Loc{SourcePath: filepath.Join(wd, "testdata"), TargetPath: filepath.Join(wd, "testdata"), MtaFilename: "xxx.yaml"} - Ω(genMakefile(&ep, &ep, &ep, nil, makefile, "", true)).Should(HaveOccurred()) + Ω(genMakefile(&ep, &ep, &ep, &ep, nil, makefile, "", true)).Should(HaveOccurred()) }) It("genMakefile testing with wrong target folder (file path)", func() { ep := dir.Loc{SourcePath: filepath.Join(wd, "testdata"), TargetPath: filepath.Join(wd, "testdata", "mta.yaml"), MtaFilename: "xxx.yaml"} - Ω(genMakefile(&ep, &ep, &ep, nil, makefile, "", true)).Should(HaveOccurred()) + Ω(genMakefile(&ep, &ep, &ep, &ep, nil, makefile, "", true)).Should(HaveOccurred()) }) It("genMakefile testing with wrong mode", func() { ep := dir.Loc{SourcePath: filepath.Join(wd, "testdata")} - Ω(genMakefile(&ep, &ep, &ep, nil, makefile, "wrongMode", true)).Should(HaveOccurred()) + Ω(genMakefile(&ep, &ep, &ep, &ep, nil, makefile, "wrongMode", true)).Should(HaveOccurred()) }) + DescribeTable("genMakefile should fail when there is a circular build dependency between modules", func(mode string) { + ep := dir.Loc{SourcePath: filepath.Join(wd, "testdata"), TargetPath: filepath.Join(wd, "testdata"), MtaFilename: "circular.yaml"} + Ω(genMakefile(&ep, &ep, &ep, &ep, nil, makefile, mode, true)).Should(HaveOccurred()) + }, + Entry("in default mode", ""), + Entry("in verbose mode", "verbose"), + ) + DescribeTable("generate module build in verbose make file", func(mtaFileName, moduleName, expectedModuleCommandsGen string) { ep := dir.Loc{SourcePath: filepath.Join(wd, "testdata", "modulegen"), TargetPath: filepath.Join(wd, "testdata"), Descriptor: "dev", MtaFilename: mtaFileName} - Ω(makeFile(&ep, &ep, nil, makeFileName, &tpl, true)).Should(Succeed()) + Ω(makeFile(&ep, &ep, &ep, nil, makeFileName, &tpl, true)).Should(Succeed()) Ω(makeFileFullPath).Should(BeAnExistingFile()) makefileContent := getMakeFileContent(makeFileFullPath) @@ -148,13 +162,40 @@ makefile_version: 0.0.0 Entry("module with commands with special characters", "commands_with_special_chars.yaml", "commands_with_special_chars", `$(MBT) execute -c='bash -c '\''echo "a"'\' -c='echo "a\b"'`), ) + + modulegen := filepath.Join(wd, "testdata", "modulegen") + DescribeTable("generate module build with dependencies in verbose make file", func(mtaFileName, moduleName, modulePath, expectedModuleDepNames string, expectedModuleDepCopyCommands string) { + ep := dir.Loc{SourcePath: modulegen, TargetPath: filepath.Join(wd, "testdata"), Descriptor: "dev", MtaFilename: mtaFileName} + Ω(makeFile(&ep, &ep, &ep, nil, makeFileName, &tpl, true)).Should(Succeed()) + Ω(makeFileFullPath).Should(BeAnExistingFile()) + makefileContent := getMakeFileContent(makeFileFullPath) + + expectedModuleGen := fmt.Sprintf(`%s: validate %s%s + @cd "$(PROJ_DIR)/%s" &&`, moduleName, expectedModuleDepNames, expectedModuleDepCopyCommands, modulePath) + Ω(makefileContent).Should(ContainSubstring(removeSpecialSymbols([]byte(expectedModuleGen)))) + }, + Entry("dependency with artifacts", "dep_with_patterns.yaml", "module1", "public", `dep`, fmt.Sprintf(` + @$(MBT) cp -s=%s -t=%s -p=dist/\* -p=some_dir -p=a\*.txt`, escapePath(modulegen, "client"), escapePath(modulegen, "public"))), + Entry("module with two dependencies", "two_deps.yaml", "my_proj_ui_deployer", "my_proj_ui_deployer", `ui5module1 ui5module2`, fmt.Sprintf(` + @$(MBT) cp -s=%s -t=%s -p=./\* + @$(MBT) cp -s=%s -t=%s -p=./\*`, + escapePath(modulegen, "ui5module1", "dist"), escapePath(modulegen, "my_proj_ui_deployer", "resources", "ui5module1"), + escapePath(modulegen, "ui5module2", "dist"), escapePath(modulegen, "my_proj_ui_deployer", "resources", "ui5module2"))), + Entry("dependency with target-path", "dep_with_artifacts_and_targetpath.yaml", "module1", "public", `module1-dep`, fmt.Sprintf(` + @$(MBT) cp -s=%s -t=%s -p=dist/\*`, escapePath(modulegen, "client"), escapePath(modulegen, "public", "client"))), + Entry("dependent module with build-result and module with artifacts and target-path", "dep_with_build_results.yaml", "module1", "public", `dep1 dep2`, fmt.Sprintf(` + @$(MBT) cp -s=%s -t=%s -p=\* + @$(MBT) cp -s=%s -t=%s -p=\*`, + escapePath(modulegen, "client1", "dist"), escapePath(modulegen, "public", "dep1_result"), + escapePath(modulegen, "client2", "target/*.war"), escapePath(modulegen, "public"))), + ) }) DescribeTable("Makefile Generation Failed", func(testPath string, testTemplateFilename string) { wd, _ := os.Getwd() testTemplate, _ := ioutil.ReadFile(filepath.Join(wd, "testdata", testTemplateFilename)) ep := dir.Loc{SourcePath: filepath.Join(wd, "testdata"), TargetPath: filepath.Join(wd, "testdata")} - Ω(makeFile(&ep, &ep, nil, makeFileName, &tplCfg{relPath: testPath, tplContent: testTemplate, preContent: basePreVerbose, postContent: basePost, depDesc: "dev"}, true)).Should(HaveOccurred()) + Ω(makeFile(&ep, &ep, &ep, nil, makeFileName, &tplCfg{relPath: testPath, tplContent: testTemplate, preContent: basePreVerbose, postContent: basePost, depDesc: "dev"}, true)).Should(HaveOccurred()) }, Entry("Wrong Template", "testdata", filepath.Join("testdata", "WrongMakeTmpl.txt")), Entry("Yaml not exists", "testdata1", "make_default.txt"), @@ -199,4 +240,25 @@ makefile_version: 0.0.0 Entry("extension paths are separated by a comma", []string{absPath("my.mtaext"), absPath("second.mtaext")}, absPath("."), ` -e="$(CURDIR)`+sep+`my.mtaext,$(CURDIR)`+sep+`second.mtaext"`), ) + + It("GetModuleDeps returns error when module doesn't exist", func() { + data := templateData{File: mta.MTA{}} + _, err := data.GetModuleDeps("unknown") + Ω(err).Should(HaveOccurred()) + }) + + It("GetModuleDeps returns error when module has dependency that doesn't exist", func() { + data := templateData{File: mta.MTA{Modules: []*mta.Module{ + { + Name: "m1", + BuildParams: map[string]interface{}{ + "requires": []interface{}{ + map[string]interface{}{"name": "unknown"}, + }, + }, + }, + }}} + _, err := data.GetModuleDeps("m1") + Ω(err).Should(HaveOccurred()) + }) }) diff --git a/internal/tpl/testdata/circular.yaml b/internal/tpl/testdata/circular.yaml new file mode 100644 index 000000000..302ed701a --- /dev/null +++ b/internal/tpl/testdata/circular.yaml @@ -0,0 +1,18 @@ +ID: mtatest +_schema-version: '3.1' +version: 0.0.1 + +modules: +- name: module1 + type: html5 + path: ui + build-parameters: + requires: + - name: module2 + +- name: module2 + type: html5 + path: ui2 + build-parameters: + requires: + - name: module1 \ No newline at end of file diff --git a/internal/tpl/testdata/modulegen/dep_with_artifacts_and_targetpath.yaml b/internal/tpl/testdata/modulegen/dep_with_artifacts_and_targetpath.yaml new file mode 100644 index 000000000..5233583ac --- /dev/null +++ b/internal/tpl/testdata/modulegen/dep_with_artifacts_and_targetpath.yaml @@ -0,0 +1,29 @@ +ID: testmta +_schema-version: '3.2' +version: 1.0.0 + +modules: + - name: module1 + type: html5 + path: public + provides: + - name: module1_feature + public: true + build-parameters: + builder: npm + ignore: [".che/", ".npmrc"] + timeout: 15m + requires: + - name: module1-dep + artifacts: ["dist/*"] + target-path: "client" + - name: module1-dep + type: html5 + path: client + build-parameters: + builder: custom + commands: + - npm install + - npm prune --production + timeout: 15m + supported-platforms: [] \ No newline at end of file diff --git a/internal/tpl/testdata/modulegen/dep_with_build_results.yaml b/internal/tpl/testdata/modulegen/dep_with_build_results.yaml new file mode 100644 index 000000000..26e2b072c --- /dev/null +++ b/internal/tpl/testdata/modulegen/dep_with_build_results.yaml @@ -0,0 +1,29 @@ +ID: testmta +_schema-version: '3.2' +version: 1.0.0 + +modules: + - name: module1 + type: html5 + path: public + build-parameters: + builder: npm + requires: + - name: dep1 + artifacts: ["*"] + target-path: "dep1_result" + - name: dep2 + artifacts: ["*"] + - name: dep1 + type: html5 + path: client1 + build-parameters: + builder: npm + supported-platforms: [] + build-result: dist + - name: dep2 + type: java + path: client2 + build-parameters: + builder: maven + supported-platforms: [] \ No newline at end of file diff --git a/internal/tpl/testdata/modulegen/dep_with_patterns.yaml b/internal/tpl/testdata/modulegen/dep_with_patterns.yaml new file mode 100644 index 000000000..0dbc54eec --- /dev/null +++ b/internal/tpl/testdata/modulegen/dep_with_patterns.yaml @@ -0,0 +1,19 @@ +ID: testmta +_schema-version: '3.2' +version: 1.0.0 + +modules: + - name: module1 + type: html5 + path: public + build-parameters: + builder: npm + requires: + - name: dep + artifacts: ["dist/*", "some_dir", "a*.txt"] + - name: dep + type: html5 + path: client + build-parameters: + builder: npm + supported-platforms: [] \ No newline at end of file diff --git a/internal/tpl/testdata/modulegen/two_deps.yaml b/internal/tpl/testdata/modulegen/two_deps.yaml new file mode 100644 index 000000000..222dcf2f7 --- /dev/null +++ b/internal/tpl/testdata/modulegen/two_deps.yaml @@ -0,0 +1,68 @@ +ID: testmta +_schema-version: '3.2' +version: 1.0.0 +parameters: + deploy_mode: html5-repo + +# Example of html5 repo with 2 html5 modules +modules: + - name: my_proj_appRouter + type: approuter.nodejs + path: my_proj_appRouter + parameters: + disk-quota: 256M + memory: 256M + requires: + - name: my_proj_html5_repo_runtime + - name: uaa_my_proj + - name: my_proj_ui_deployer + type: com.sap.html5.application-content + path: my_proj_ui_deployer + requires: + - name: my_proj_html5_repo_host + build-parameters: + requires: + - name: ui5module1 + artifacts: + - './*' + target-path: resources/ui5module1 + - name: ui5module2 + artifacts: + - './*' + target-path: resources/ui5module2 + - name: ui5module1 + type: html5 + path: ui5module1 + build-parameters: + builder: grunt + supported-platforms: [] + build-result: dist + - name: ui5module2 + type: html5 + path: ui5module2 + build-parameters: + builder: grunt + supported-platforms: [] + build-result: dist +resources: + - name: my_proj_html5_repo_runtime + parameters: + service-plan: app-runtime + service: html5-apps-repo + type: org.cloudfoundry.managed-service + - name: my_proj_html5_repo_host + parameters: + service-plan: app-host + service: html5-apps-repo + type: org.cloudfoundry.managed-service + - name: uaa_my_proj + parameters: + path: ./xs-security.json + service-plan: application + service: xsuaa + type: org.cloudfoundry.managed-service + - name: dest_my_proj + parameters: + service-plan: lite + service: destination + type: org.cloudfoundry.managed-service