Skip to content
Permalink
Browse files

cmd/go: add //go:embed support

The final piece of //go:embed support: have the go command stitch
together parsing in go/build, low-level data initialization in cmd/compile,
and the new data structures in package embed, to make the //go:embed
feature actually function.

And test, now that all the pieces are available to work together.

For #41191.
(Issue not fixed: still need to add a tool for use by Bazel.)

Change-Id: Ib1d198345c3b4d557d340f292eda13b984b65d65
Reviewed-on: https://go-review.googlesource.com/c/go/+/243945
Trust: Russ Cox <rsc@golang.org>
Trust: Jay Conrod <jayconrod@google.com>
Trust: Johan Brandhorst <johan.brandhorst@gmail.com>
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>
Reviewed-by: Johan Brandhorst <johan.brandhorst@gmail.com>
  • Loading branch information
rsc committed Oct 29, 2020
1 parent ddc7e1d commit 25d28ec55aded46e0be9c2298f24287d296a9e47
@@ -426,7 +426,7 @@ func walk(path string, info fs.FileInfo, walkFn filepath.WalkFunc) error {
// Walk walks the file tree rooted at root, calling walkFn for each file or
// directory in the tree, including root.
func Walk(root string, walkFn filepath.WalkFunc) error {
info, err := lstat(root)
info, err := Lstat(root)
if err != nil {
err = walkFn(root, nil, err)
} else {
@@ -439,7 +439,7 @@ func Walk(root string, walkFn filepath.WalkFunc) error {
}

// lstat implements a version of os.Lstat that operates on the overlay filesystem.

This comment has been minimized.

Copy link
@rtkkroland

rtkkroland Oct 29, 2020

Comment needs upating to match code

This comment has been minimized.

Copy link
@ianlancetaylor

ianlancetaylor Oct 29, 2020

Contributor

Thanks. We use Gerrit for code review, and mirror to GitHub. Very few people see comments on the changes on GitHub. Would you mind adding your comment to https://golang.org/cl/243945? Thanks.

This comment has been minimized.

Copy link
@rtkkroland

rtkkroland Oct 29, 2020

Done. Thanks. 👍

func lstat(path string) (fs.FileInfo, error) {
func Lstat(path string) (fs.FileInfo, error) {
return overlayStat(path, os.Lstat, "lstat")
}

@@ -523,7 +523,7 @@ func Glob(pattern string) (matches []string, err error) {
return nil, err
}
if !hasMeta(pattern) {
if _, err = lstat(pattern); err != nil {
if _, err = Lstat(pattern); err != nil {
return nil, nil
}
return []string{pattern}, nil
@@ -918,7 +918,7 @@ contents`,
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
initOverlay(t, tc.overlay)
got, err := lstat(tc.path)
got, err := Lstat(tc.path)
if tc.wantErr {
if err == nil {
t.Errorf("lstat(%q): got no error, want error", tc.path)
@@ -570,6 +570,8 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
// Show vendor-expanded paths in listing
p.TestImports = p.Resolve(p.TestImports)
p.XTestImports = p.Resolve(p.XTestImports)
p.TestEmbedFiles = p.ResolveEmbed(p.TestEmbedPatterns)
p.XTestEmbedFiles = p.ResolveEmbed(p.XTestEmbedPatterns)
p.DepOnly = !cmdline[p]

if *listCompiled {
@@ -17,6 +17,7 @@ import (
"io/fs"
"io/ioutil"
"os"
"path"
pathpkg "path"
"path/filepath"
"runtime"
@@ -94,6 +95,10 @@ type PackagePublic struct {
SwigCXXFiles []string `json:",omitempty"` // .swigcxx files
SysoFiles []string `json:",omitempty"` // .syso system object files added to package

// Embedded files
EmbedPatterns []string `json:",omitempty"` // //go:embed patterns
EmbedFiles []string `json:",omitempty"` // files and directories matched by EmbedPatterns

// Cgo directives
CgoCFLAGS []string `json:",omitempty"` // cgo: flags for C compiler
CgoCPPFLAGS []string `json:",omitempty"` // cgo: flags for C preprocessor
@@ -115,10 +120,14 @@ type PackagePublic struct {
// Test information
// If you add to this list you MUST add to p.AllFiles (below) too.
// Otherwise file name security lists will not apply to any new additions.
TestGoFiles []string `json:",omitempty"` // _test.go files in package
TestImports []string `json:",omitempty"` // imports from TestGoFiles
XTestGoFiles []string `json:",omitempty"` // _test.go files outside package
XTestImports []string `json:",omitempty"` // imports from XTestGoFiles
TestGoFiles []string `json:",omitempty"` // _test.go files in package
TestImports []string `json:",omitempty"` // imports from TestGoFiles
TestEmbedPatterns []string `json:",omitempty"` // //go:embed patterns
TestEmbedFiles []string `json:",omitempty"` // //files matched by EmbedPatterns
XTestGoFiles []string `json:",omitempty"` // _test.go files outside package
XTestImports []string `json:",omitempty"` // imports from XTestGoFiles
XTestEmbedPatterns []string `json:",omitempty"` // //go:embed patterns
XTestEmbedFiles []string `json:",omitempty"` // //files matched by EmbedPatterns
}

// AllFiles returns the names of all the files considered for the package.
@@ -127,7 +136,7 @@ type PackagePublic struct {
// The go/build package filtered others out (like foo_wrongGOARCH.s)
// and that's OK.
func (p *Package) AllFiles() []string {
return str.StringList(
files := str.StringList(
p.GoFiles,
p.CgoFiles,
// no p.CompiledGoFiles, because they are from GoFiles or generated by us
@@ -145,6 +154,27 @@ func (p *Package) AllFiles() []string {
p.TestGoFiles,
p.XTestGoFiles,
)

// EmbedFiles may overlap with the other files.
// Dedup, but delay building the map as long as possible.
// Only files in the current directory (no slash in name)
// need to be checked against the files variable above.
var have map[string]bool
for _, file := range p.EmbedFiles {
if !strings.Contains(file, "/") {
if have == nil {
have = make(map[string]bool)
for _, file := range files {
have[file] = true
}
}
if have[file] {
continue
}
}
files = append(files, file)
}
return files
}

// Desc returns the package "description", for use in b.showOutput.
@@ -174,6 +204,7 @@ type PackageInternal struct {
GobinSubdir bool // install target would be subdir of GOBIN
BuildInfo string // add this info to package main
TestmainGo *[]byte // content for _testmain.go
Embed map[string][]string // //go:embed comment mapping

Asmflags []string // -asmflags for this package
Gcflags []string // -gcflags for this package
@@ -366,6 +397,9 @@ func (p *Package) copyBuild(pp *build.Package) {
p.TestImports = nil
p.XTestImports = nil
}
p.EmbedPatterns = pp.EmbedPatterns
p.TestEmbedPatterns = pp.TestEmbedPatterns
p.XTestEmbedPatterns = pp.XTestEmbedPatterns
}

// A PackageError describes an error loading information about a package.
@@ -960,6 +994,12 @@ func (pre *preload) preloadImports(imports []string, parent *build.Package) {
// loadPackageData have completed. The preloader will not make any new calls
// to loadPackageData.
func (pre *preload) flush() {
// flush is usually deferred.
// Don't hang program waiting for workers on panic.
if v := recover(); v != nil {
panic(v)
}

close(pre.cancel)
for i := 0; i < preloadWorkerCount; i++ {
pre.sema <- struct{}{}
@@ -1624,6 +1664,11 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor
p.setLoadPackageDataError(err, path, stk, importPos)
}

p.EmbedFiles, p.Internal.Embed, err = p.resolveEmbed(p.EmbedPatterns)
if err != nil {
setError(err)
}

useBindir := p.Name == "main"
if !p.Standard {
switch cfg.BuildBuildmode {
@@ -1865,6 +1910,153 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor
}
}

// ResolveEmbed resolves //go:embed patterns and returns only the file list.
// For use by go list to compute p.TestEmbedFiles and p.XTestEmbedFiles.
func (p *Package) ResolveEmbed(patterns []string) []string {
files, _, _ := p.resolveEmbed(patterns)
return files
}

// resolveEmbed resolves //go:embed patterns to precise file lists.
// It sets files to the list of unique files matched (for go list),
// and it sets pmap to the more precise mapping from
// patterns to files.
// TODO(rsc): All these messages need position information for better error reports.
func (p *Package) resolveEmbed(patterns []string) (files []string, pmap map[string][]string, err error) {
pmap = make(map[string][]string)
have := make(map[string]int)
dirOK := make(map[string]bool)
pid := 0 // pattern ID, to allow reuse of have map
for _, pattern := range patterns {
pid++

// Check pattern is valid for //go:embed.
if _, err := path.Match(pattern, ""); err != nil || !validEmbedPattern(pattern) {
return nil, nil, fmt.Errorf("pattern %s: invalid pattern syntax", pattern)
}

// Glob to find matches.
match, err := fsys.Glob(p.Dir + string(filepath.Separator) + filepath.FromSlash(pattern))
if err != nil {
return nil, nil, fmt.Errorf("pattern %s: %v", pattern, err)
}

// Filter list of matches down to the ones that will still exist when
// the directory is packaged up as a module. (If p.Dir is in the module cache,
// only those files exist already, but if p.Dir is in the current module,
// then there may be other things lying around, like symbolic links or .git directories.)
var list []string
for _, file := range match {
rel := filepath.ToSlash(file[len(p.Dir)+1:]) // file, relative to p.Dir

what := "file"
info, err := fsys.Lstat(file)
if err != nil {
return nil, nil, err
}
if info.IsDir() {
what = "directory"
}

// Check that directories along path do not begin a new module
// (do not contain a go.mod).
for dir := file; len(dir) > len(p.Dir)+1 && !dirOK[dir]; dir = filepath.Dir(dir) {
if _, err := fsys.Stat(filepath.Join(dir, "go.mod")); err == nil {
return nil, nil, fmt.Errorf("pattern %s: cannot embed %s %s: in different module", pattern, what, rel)
}
if dir != file {
if info, err := fsys.Lstat(dir); err == nil && !info.IsDir() {
return nil, nil, fmt.Errorf("pattern %s: cannot embed %s %s: in non-directory %s", pattern, what, rel, dir[len(p.Dir)+1:])
}
}
dirOK[dir] = true
if elem := filepath.Base(dir); isBadEmbedName(elem) {
if dir == file {
return nil, nil, fmt.Errorf("pattern %s: cannot embed %s %s: invalid name %s", pattern, what, rel, elem)
} else {
return nil, nil, fmt.Errorf("pattern %s: cannot embed %s %s: in invalid directory %s", pattern, what, rel, elem)
}
}
}

switch {
default:
return nil, nil, fmt.Errorf("pattern %s: cannot embed irregular file %s", pattern, rel)

case info.Mode().IsRegular():
if have[rel] != pid {
have[rel] = pid
list = append(list, rel)
}

case info.IsDir():
// Gather all files in the named directory, stopping at module boundaries
// and ignoring files that wouldn't be packaged into a module.
count := 0
err := fsys.Walk(file, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
rel := filepath.ToSlash(path[len(p.Dir)+1:])
if info.IsDir() {
if _, err := fsys.Stat(filepath.Join(path, "go.mod")); err == nil {
return filepath.SkipDir
}
return nil
}
if !info.Mode().IsRegular() {
return nil
}
if isBadEmbedName(info.Name()) {
// Ignore bad names, assuming they won't go into modules.
return nil
}
count++
if have[rel] != pid {
have[rel] = pid
list = append(list, rel)
}
return nil
})
if err != nil {
return nil, nil, err
}
if count == 0 {
return nil, nil, fmt.Errorf("pattern %s: cannot embed directory %s: contains no embeddable files", pattern, rel)
}
}
}

if len(list) == 0 {
return nil, nil, fmt.Errorf("pattern %s: no matching files found", pattern)
}
sort.Strings(list)
pmap[pattern] = list
}

for file := range have {
files = append(files, file)
}
sort.Strings(files)
return files, pmap, nil
}

func validEmbedPattern(pattern string) bool {
return pattern != "." && fs.ValidPath(pattern)
}

// isBadEmbedName reports whether name is the base name of a file that
// can't or won't be included in modules and therefore shouldn't be treated
// as existing for embedding.
func isBadEmbedName(name string) bool {
switch name {
// Version control directories won't be present in module.
case ".bzr", ".hg", ".git", ".svn":
return true
}
return false
}

// collectDeps populates p.Deps and p.DepsErrors by iterating over
// p.Internal.Imports.
//
@@ -105,6 +105,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
var ptestErr, pxtestErr *PackageError
var imports, ximports []*Package
var stk ImportStack
var testEmbed, xtestEmbed map[string][]string
stk.Push(p.ImportPath + " (test)")
rawTestImports := str.StringList(p.TestImports)
for i, path := range p.TestImports {
@@ -122,7 +123,16 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
p.TestImports[i] = p1.ImportPath
imports = append(imports, p1)
}
var err error
p.TestEmbedFiles, testEmbed, err = p.resolveEmbed(p.TestEmbedPatterns)
if err != nil && ptestErr == nil {
ptestErr = &PackageError{
ImportStack: stk.Copy(),
Err: err,
}
}
stk.Pop()

stk.Push(p.ImportPath + "_test")
pxtestNeedsPtest := false
rawXTestImports := str.StringList(p.XTestImports)
@@ -135,6 +145,13 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
}
p.XTestImports[i] = p1.ImportPath
}
p.XTestEmbedFiles, xtestEmbed, err = p.resolveEmbed(p.XTestEmbedPatterns)
if err != nil && pxtestErr == nil {
pxtestErr = &PackageError{
ImportStack: stk.Copy(),
Err: err,
}
}
stk.Pop()

// Test package.
@@ -174,6 +191,14 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
m[k] = append(m[k], v...)
}
ptest.Internal.Build.ImportPos = m
if testEmbed == nil && len(p.Internal.Embed) > 0 {
testEmbed = map[string][]string{}
}
for k, v := range p.Internal.Embed {
testEmbed[k] = v
}
ptest.Internal.Embed = testEmbed
ptest.EmbedFiles = str.StringList(p.EmbedFiles, p.TestEmbedFiles)
ptest.collectDeps()
} else {
ptest = p
@@ -193,6 +218,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
ForTest: p.ImportPath,
Module: p.Module,
Error: pxtestErr,
EmbedFiles: p.XTestEmbedFiles,
},
Internal: PackageInternal{
LocalPrefix: p.Internal.LocalPrefix,
@@ -206,6 +232,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p
Gcflags: p.Internal.Gcflags,
Ldflags: p.Internal.Ldflags,
Gccgoflags: p.Internal.Gccgoflags,
Embed: xtestEmbed,
},
}
if pxtestNeedsPtest {

0 comments on commit 25d28ec

Please sign in to comment.
You can’t perform that action at this time.