internal/mod/modimports: include files from parent directories

CUE supports packages that span multiple directories.
When determining the set of packages to load, we need
to consider parent directories too.

Also change the `AllModuleFiles` logic so that it guarantees that
all the files in a directory will be adjacent in the iteration.
This means that a consumer needs to maintain less state
when doing per-package logic - it can discard any state
for a directory when it see that the iteration has moved to
a different directory.

For #3155

Signed-off-by: Roger Peppe <rogpeppe@gmail.com>
Change-Id: Id342ccc9f4505cfcfb183dde49e8aed71a1fe596
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1195280
Unity-Result: CUE porcuepine <cue.porcuepine@gmail.com>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>

internal/mod/modimports/modimports.go

@@ -68,23 +68,78 @@ func AllImports(modFilesIter func(func(ModuleFile, error) bool)) (_ []string, re
 // If pkgQualifier is "*", files from all packages in the directory will be produced.
 func PackageFiles(fsys fs.FS, dir string, pkgQualifier string) func(func(ModuleFile, error) bool) {
 	return func(yield func(ModuleFile, error) bool) {
-		entries, err := fs.ReadDir(fsys, dir)
-		if err != nil {
-			yield(ModuleFile{
-				FilePath: dir,
-			}, err)
-			return
+		// Start at the target directory, but also include package files
+		// from packages with the same name(s) in parent directories.
+		// Stop the iteration when we find a cue.mod entry, signifying
+		// the module root. If the location is inside a `cue.mod` directory
+		// already, do not look at parent directories - this mimics historic
+		// behavior.
+		selectPackage := func(pkg string) bool {
+			if pkgQualifier == "*" {
+				return true
+			}
+			return pkg == pkgQualifier
+		}
+		inCUEMod := false
+		if before, after, ok := strings.Cut(dir, "cue.mod"); ok {
+			// We're underneath a cue.mod directory if some parent
+			// element is cue.mod.
+			inCUEMod =
+				(before == "" || strings.HasSuffix(before, "/")) &&
+					(after == "" || strings.HasPrefix(after, "/"))
 		}
-		for _, e := range entries {
-			if !yieldPackageFile(fsys, path.Join(dir, e.Name()), pkgQualifier, yield) {
+		var matchedPackages map[string]bool
+		for {
+			entries, err := fs.ReadDir(fsys, dir)
+			if err != nil {
+				yield(ModuleFile{
+					FilePath: dir,
+				}, err)
+				return
+			}
+			inModRoot := false
+			for _, e := range entries {
+				if e.Name() == "cue.mod" {
+					inModRoot = true
+				}
+				pkgName, cont := yieldPackageFile(fsys, path.Join(dir, e.Name()), selectPackage, yield)
+				if !cont {
+					return
+				}
+				if pkgName != "" {
+					if matchedPackages == nil {
+						matchedPackages = make(map[string]bool)
+					}
+					matchedPackages[pkgName] = true
+				}
+			}
+			if inModRoot || inCUEMod {
+				// We're at the module root or we're inside the cue.mod
+				// directory. Don't go any further up the hierarchy.
 				return
 			}
+			if matchedPackages == nil {
+				// No packages possible in parent directories if there are
+				// no matching package files in the package directory itself.
+				return
+			}
+			selectPackage = func(pkgName string) bool {
+				return matchedPackages[pkgName]
+			}
+			parent := path.Dir(dir)
+			if len(parent) >= len(dir) {
+				// No more parent directories.
+				return
+			}
+			dir = parent
 		}
 	}
 }
 
 // AllModuleFiles returns an iterator that produces all the CUE files inside the
 // module at the given root.
+//
+// The caller may assume that files from the same package are always adjacent.
 func AllModuleFiles(fsys fs.FS, root string) func(func(ModuleFile, error) bool) {
 	return func(yield func(ModuleFile, error) bool) {
 		yieldAllModFiles(fsys, root, true, yield)
@@ -115,33 +170,50 @@ func yieldAllModFiles(fsys fs.FS, fpath string, topDir bool, yield func(ModuleFi
 			}
 		}
 	}
+	// Generate all entries for the package before moving onto packages
+	// in subdirectories.
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+		fpath := path.Join(fpath, entry.Name())
+		if _, ok := yieldPackageFile(fsys, fpath, func(string) bool { return true }, yield); !ok {
+			return false
+		}
+	}
+
 	for _, entry := range entries {
 		name := entry.Name()
+		if !entry.IsDir() {
+			continue
+		}
+		if name == "cue.mod" || strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") {
+			continue
+		}
 		fpath := path.Join(fpath, name)
-		if entry.IsDir() {
-			if name == "cue.mod" || strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") {
-				continue
-			}
-			if !yieldAllModFiles(fsys, fpath, false, yield) {
-				return false
-			}
-		} else if !yieldPackageFile(fsys, fpath, "*", yield) {
+		if !yieldAllModFiles(fsys, fpath, false, yield) {
 			return false
 		}
 	}
 	return true
 }
 
-func yieldPackageFile(fsys fs.FS, fpath, pkgQualifier string, yield func(ModuleFile, error) bool) bool {
+// yieldPackageFile invokes yield with the contents of the package file
+// at the given path if selectPackage returns true for the file's
+// package name.
+//
+// It returns the yielded package name (if any) and reports whether
+// the iteration should continue.
+func yieldPackageFile(fsys fs.FS, fpath string, selectPackage func(pkgName string) bool, yield func(ModuleFile, error) bool) (pkgName string, cont bool) {
 	if !strings.HasSuffix(fpath, ".cue") {
-		return true
+		return "", true
 	}
 	pf := ModuleFile{
 		FilePath: fpath,
 	}
 	f, err := fsys.Open(fpath)
 	if err != nil {
-		return yield(pf, err)
+		return "", yield(pf, err)
 	}
 	defer f.Close()
 
@@ -152,16 +224,16 @@ func yieldPackageFile(fsys fs.FS, fpath, pkgQualifier string, yield func(ModuleF
 	// on a reader in a streaming manner.
 	data, err := cueimports.Read(f)
 	if err != nil {
-		return yield(pf, err)
+		return "", yield(pf, err)
 	}
 	syntax, err := parser.ParseFile(fpath, data, parser.ImportsOnly)
 	if err != nil {
-		return yield(pf, err)
+		return "", yield(pf, err)
 	}
 
-	if pkgQualifier != "*" && syntax.PackageName() != pkgQualifier {
-		return true
+	if !selectPackage(syntax.PackageName()) {
+		return "", true
 	}
 	pf.Syntax = syntax
-	return yield(pf, nil)
+	return syntax.PackageName(), yield(pf, nil)
 }

internal/mod/modimports/modimports_test.go

@@ -118,5 +118,5 @@ import (
 	qt.Assert(t, qt.DeepEquals(imps, []string{"other"}))
 	imps, err = AllImports(PackageFiles(tfs, "sub", "x"))
 	qt.Assert(t, qt.IsNil(err))
-	qt.Assert(t, qt.DeepEquals(imps, []string{"imported-from-sub.com/foo"}))
+	qt.Assert(t, qt.DeepEquals(imps, []string{"bar.com/baz", "foo", "imported-from-sub.com/foo", "something.else:other"}))
 }

internal/mod/modimports/testdata/simple.txtar

@@ -1,10 +1,10 @@
 -- want --
 x.cue "dep1" "dep2:a" "dep2:b" "dep3"
+z.cue
 y/y1.cue "dep3" "dep4"
 y/y2.cue
 y/z1.cue
 y/z2.cue
-z.cue
 -- want-imports --
 dep1
 dep2:a

internal/mod/modpkgload/testdata/multidir.txtar

@@ -15,6 +15,12 @@ example.com/blah@v0
 	flags: inAll,isRoot,fromRoot,importsLoaded
 	mod: example.com@v0.0.1
 	location: _registry/example.com_v0.0.1/blah
+	imports:
+		foo.com/bar/hello/goodbye@v0
+foo.com/bar/hello/goodbye@v0
+	flags: inAll,isRoot,fromRoot
+	error: cannot find module providing package foo.com/bar/hello/goodbye@v0
+	missing: true
 -- main.cue --
 package main
 import "example.com/blah@v0"