Update for fixing vendoring issue

from: revel#218
wiselike · Apr 5, 2024 · 9b690fd · 9b690fd
1 parent 4c7ddf5
commit 9b690fd
Show file tree

Hide file tree

Showing 4 changed files with 209 additions and 5 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,8 +1,8 @@
 language: go
 
 go:
-  - "1.13.x"
-  - "1.14.x"
+  - "1.18.x"
+  - "1.17.x"
   - "tip"
 
 os:
@@ -66,6 +66,14 @@ script:
   - revel new     --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@$REVEL_BRANCH" -a my/testapp3 --package revelframework.com
   - revel test    --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@$REVEL_BRANCH" -a my/testapp3
 
+  # Check vendored version of revel
+  - cd $INITIALWD
+  - revel new     --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@$REVEL_BRANCH" -a my/testapp4 --package revelframework.com
+  - cd my/testapp4
+  - go mod vendor
+  - cd ../..
+  - revel test    --gomod-flags "edit -replace=github.com/revel/revel=github.com/revel/revel@$REVEL_BRANCH" -a my/testapp4
+
 matrix:
   allow_failures:
     - go: tip

diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -6,7 +6,7 @@
         {
             "label": "Clean-Test-Project",
             "type": "shell",
-            "command": "rm -rf ${workspaceRoot}/.temp/revel/reveltest"
+            "command": "rm -rf ${workspaceRoot}/.temp/revel/reveltest && mkdir -p ${workspaceRoot}/.temp/revel/GOPATH"
         },
         {
             "label": "Update Go Mod",

diff --git a/model/revel_container.go b/model/revel_container.go
@@ -2,10 +2,16 @@
 package model
 
 import (
+	"bufio"
 	"fmt"
+	"go/build"
+	"io"
+	"io/fs"
+	"os"
 	"path/filepath"
 	"sort"
 	"strings"
+	"unicode"
 
 	"github.com/revel/cmd/utils"
 	"github.com/revel/config"
@@ -93,6 +99,15 @@ type (
 		FireEventFunction func(key Event, value interface{}) (response EventResponse)
 		ImportFunction    func(pkgName string) error
 	}
+	Mod struct {
+		ImportPath    string
+		SourcePath    string
+		Version       string
+		SourceVersion string
+		Dir           string   // full path, $GOPATH/pkg/mod/
+		Pkgs          []string // sub-pkg import paths
+		VendorList    []string // files to vendor
+	}
 )
 
 // Simple Wrapped RevelCallback.
@@ -235,15 +250,40 @@ func (rp *RevelContainer) loadModules(callback RevelCallback) (err error) {
 
 	// Reorder module order by key name, a poor mans sort but at least it is consistent
 	sort.Strings(keys)
+	modtxtPath := filepath.Join(rp.SourcePath, "vendor", "modules.txt")
+	if utils.Exists(modtxtPath) {
+		// Parse out require sections of module.txt
+		modules := rp.vendorInitilizeLocal(modtxtPath, keys)
+		for _, mod := range modules {
+			for _, vendorFile := range mod.VendorList {
+				x := strings.Index(vendorFile, mod.Dir)
+				if x < 0 {
+					utils.Logger.Crit("Error! vendor file doesn't belong to mod, strange.", "vendorFile", "mod.Dir", mod.Dir)
+				}
+
+				localPath := fmt.Sprintf("%s%s", mod.ImportPath, vendorFile[len(mod.Dir):])
+				localFile := filepath.Join(rp.SourcePath, "vendor", localPath)
+
+				utils.Logger.Infof("vendoring %s\n", localPath)
+
+				os.MkdirAll(filepath.Dir(localFile), os.ModePerm)
+				if _, err := copyFile(vendorFile, localFile); err != nil {
+					fmt.Printf("Error! %s - unable to copy file %s\n", err.Error(), vendorFile)
+					os.Exit(1)
+				}
+			}
+		}
+	}
+
 	for _, key := range keys {
 		moduleImportPath := rp.Config.StringDefault(key, "")
 		if moduleImportPath == "" {
 			continue
 		}
 
 		modulePath, err := rp.ResolveImportPath(moduleImportPath)
+		utils.Logger.Info("Resolving import path ", "modulePath", modulePath, "module_import_path", moduleImportPath, "error", err)
 		if err != nil {
-			utils.Logger.Info("Missing module ", "module_import_path", moduleImportPath, "error", err)
 
 			if err := callback.PackageResolver(moduleImportPath); err != nil {
 				return fmt.Errorf("failed to resolve package %w", err)
@@ -266,6 +306,88 @@ func (rp *RevelContainer) loadModules(callback RevelCallback) (err error) {
 	return
 }
 
+// Adds a module paths to the container object.
+func (rp *RevelContainer) vendorInitilizeLocal(modtxtPath string, revel_modules_keys []string) []*Mod {
+	revel_modules := []string{"github.com/revel/revel"}
+	for _, key := range revel_modules_keys {
+		moduleImportPath := rp.Config.StringDefault(key, "")
+		if moduleImportPath == "" {
+			continue
+		}
+		revel_modules = append(revel_modules, moduleImportPath)
+	}
+	f, _ := os.Open(modtxtPath)
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	scanner.Split(bufio.ScanLines)
+
+	var (
+		mod *Mod
+		err error
+	)
+	modules := []*Mod{}
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		// Look for # character
+		if line[0] == 35 {
+			s := strings.Split(line, " ")
+			if (len(s) != 6 && len(s) != 3) || s[1] == "explicit" {
+				continue
+			}
+
+			mod = &Mod{
+				ImportPath: s[1],
+				Version:    s[2],
+			}
+			if s[2] == "=>" {
+				// issue https://github.com/golang/go/issues/33848 added these,
+				// see comments. I think we can get away with ignoring them.
+				continue
+			}
+			// Handle "replace" in module file if any
+			if len(s) > 3 && s[3] == "=>" {
+				mod.SourcePath = s[4]
+
+				// Handle replaces with a relative target. For example:
+				// "replace github.com/status-im/status-go/protocol => ./protocol"
+				if strings.HasPrefix(s[4], ".") || strings.HasPrefix(s[4], "/") {
+					mod.Dir, err = filepath.Abs(s[4])
+					if err != nil {
+						fmt.Printf("invalid relative path: %v", err)
+						os.Exit(1)
+					}
+				} else {
+					mod.SourceVersion = s[5]
+					mod.Dir = pkgModPath(mod.SourcePath, mod.SourceVersion)
+				}
+			} else {
+				mod.Dir = pkgModPath(mod.ImportPath, mod.Version)
+			}
+
+			if _, err := os.Stat(mod.Dir); os.IsNotExist(err) {
+				utils.Logger.Critf("Error! %q module path does not exist, check $GOPATH/pkg/mod\n", mod.Dir)
+			}
+
+			// Determine if we need to examine this mod, based on the list of modules being imported
+			for _, importPath := range revel_modules {
+				if strings.HasPrefix(importPath, mod.ImportPath) {
+					updateModVendorList(mod, importPath)
+				}
+			}
+
+			// Build list of files to module path source to project vendor folder
+			modules = append(modules, mod)
+
+			continue
+		}
+
+		mod.Pkgs = append(mod.Pkgs, line)
+	}
+	return modules
+}
+
 // Adds a module paths to the container object.
 func (rp *RevelContainer) addModulePaths(name, importPath, modulePath string) {
 	utils.Logger.Info("Adding module path", "name", name, "import path", importPath, "system path", modulePath)
@@ -313,3 +435,76 @@ func (rp *RevelContainer) ResolveImportPath(importPath string) (string, error) {
 	}
 	return pkgs[0].PkgPath, fmt.Errorf("%w: %s", ErrNoFiles, importPath)
 }
+
+func normString(str string) (normStr string) {
+	for _, char := range str {
+		if unicode.IsUpper(char) {
+			normStr += "!" + string(unicode.ToLower(char))
+		} else {
+			normStr += string(char)
+		}
+	}
+	return
+}
+
+func pkgModPath(importPath, version string) string {
+	goPath := build.Default.GOPATH
+	if goPath == "" {
+		if goPath = os.Getenv("GOPATH"); goPath == "" {
+			// the default GOPATH for go v1.11
+			goPath = filepath.Join(os.Getenv("HOME"), "go")
+		}
+	}
+
+	normPath := normString(importPath)
+	normVersion := normString(version)
+
+	return filepath.Join(goPath, "pkg", "mod", fmt.Sprintf("%s@%s", normPath, normVersion))
+}
+
+func copyFile(src, dst string) (int64, error) {
+	srcStat, err := os.Stat(src)
+	if err != nil {
+		return 0, err
+	}
+
+	if !srcStat.Mode().IsRegular() {
+		return 0, fmt.Errorf("%s is not a regular file", src)
+	}
+
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return 0, err
+	}
+	defer srcFile.Close()
+
+	dstFile, err := os.Create(dst)
+	if err != nil {
+		return 0, err
+	}
+	defer dstFile.Close()
+
+	return io.Copy(dstFile, srcFile)
+}
+
+func updateModVendorList(mod *Mod, importPath string) {
+	vendorList := []string{}
+	pathPrefix := filepath.Join(mod.Dir, importPath[len(mod.ImportPath):])
+
+	filepath.WalkDir(pathPrefix, func(path string, d fs.DirEntry, err error) (e error) {
+		if d.IsDir() {
+			return
+		}
+
+		if err != nil {
+			utils.Logger.Crit("Failed to walk vendor dir")
+		}
+		utils.Logger.Info("Adding to file in vendor list", "path", path)
+		vendorList = append(vendorList, path)
+		return
+	})
+
+	utils.Logger.Info("For module", "module", mod.ImportPath, "files", len(vendorList))
+
+	mod.VendorList = append(mod.VendorList, vendorList...)
+}
diff --git a/parser2/source_processor.go b/parser2/source_processor.go
@@ -128,7 +128,8 @@ func (s *SourceProcessor) processPath(path string, info os.FileInfo, err error)
 	}
 
 	// Ignore files and folders not marked tmp (since those are generated)
-	if !info.IsDir() || info.Name() == "tmp" {
+	// Also ignore files in the vendor folder.
+	if !info.IsDir() || info.Name() == "tmp" || strings.HasPrefix(path, filepath.Join(s.revelContainer.BasePath, "vendor")) {
 		return nil
 	}