Skip to content

Commit

Permalink
hugofs: Fix mount with hole regression
Browse files Browse the repository at this point in the history 
Fixes #6854
  • Loading branch information
@bep
bep committed Feb 9, 2020
1 parent 18888e0 commit b78576f
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 10 deletions.
18 changes: 17 additions & 1 deletion hugofs/decorators.go
View file
Expand Up @@ -81,12 +81,28 @@ func DecorateBasePathFs(base *afero.BasePathFs) afero.Fs {
// NewBaseFileDecorator decorates the given Fs to provide the real filename
// and an Opener func.
func NewBaseFileDecorator(fs afero.Fs) afero.Fs {

ffs := &baseFileDecoratorFs{Fs: fs}

decorator := func(fi os.FileInfo, filename string) (os.FileInfo, error) {
// Store away the original in case it's a symlink.
meta := FileMeta{metaKeyName: fi.Name()}
if fi.IsDir() {
meta[metaKeyJoinStat] = func(name string) (FileMetaInfo, error) {
joinedFilename := filepath.Join(filename, name)
fi, _, err := lstatIfPossible(fs, joinedFilename)
if err != nil {
return nil, err
}

fi, err = ffs.decorate(fi, joinedFilename)
if err != nil {
return nil, err
}

return fi.(FileMetaInfo), nil
}
}

isSymlink := isSymlink(fi)
if isSymlink {
meta[metaKeyOriginalFilename] = filename
Expand Down
9 changes: 9 additions & 0 deletions hugofs/fileinfo.go
View file
Expand Up @@ -50,6 +50,7 @@ const (
metaKeyOpener = "opener"
metaKeyIsOrdered = "isOrdered"
metaKeyIsSymlink = "isSymlink"
metaKeyJoinStat = "joinStat"
metaKeySkipDir = "skipDir"
metaKeyClassifier = "classifier"
metaKeyTranslationBaseName = "translationBaseName"
Expand Down Expand Up @@ -177,6 +178,14 @@ func (f FileMeta) Open() (afero.File, error) {
return v.(func() (afero.File, error))()
}

func (f FileMeta) JoinStat(name string) (FileMetaInfo, error) {
v, found := f[metaKeyJoinStat]
if !found {
return nil, os.ErrNotExist
}
return v.(func(name string) (FileMetaInfo, error))(name)
}

func (f FileMeta) stringV(key string) string {
if v, found := f[key]; found {
return v.(string)
Expand Down
56 changes: 50 additions & 6 deletions hugofs/rootmapping_fs.go
View file
Expand Up @@ -128,6 +128,11 @@ type RootMapping struct {

}

type keyRootMappings struct {
key string
roots []RootMapping
}

func (rm *RootMapping) clean() {
rm.From = strings.Trim(filepath.Clean(rm.From), filepathSeparator)
rm.To = filepath.Clean(rm.To)
Expand Down Expand Up @@ -281,6 +286,21 @@ func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping {
return roots
}

func (fs *RootMappingFs) getAncestors(prefix string) []keyRootMappings {
var roots []keyRootMappings
fs.rootMapToReal.WalkPath(prefix, func(s string, v interface{}) bool {
if strings.HasPrefix(prefix, s+filepathSeparator) {
roots = append(roots, keyRootMappings{
key: s,
roots: v.([]RootMapping),
})
}
return false
})

return roots
}

func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {
meta := fis[0].Meta()
f, err := meta.Open()
Expand Down Expand Up @@ -342,17 +362,15 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error)
seen := make(map[string]bool) // Prevent duplicate directories
level := strings.Count(prefix, filepathSeparator)

// First add any real files/directories.
rms := fs.getRoot(prefix)
for _, rm := range rms {
f, err := rm.fi.Meta().Open()
collectDir := func(rm RootMapping, fi FileMetaInfo) error {
f, err := fi.Meta().Open()
if err != nil {
return nil, err
return err
}
direntries, err := f.Readdir(-1)
if err != nil {
f.Close()
return nil, err
return err
}

for _, fi := range direntries {
Expand All @@ -374,6 +392,16 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error)
}

f.Close()

return nil
}

// First add any real files/directories.
rms := fs.getRoot(prefix)
for _, rm := range rms {
if err := collectDir(rm, rm.fi); err != nil {
return nil, err
}
}

// Next add any file mounts inside the given directory.
Expand Down Expand Up @@ -428,6 +456,22 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error)
return false
})

// Finally add any ancestor dirs with files in this directory.
ancestors := fs.getAncestors(prefix)
for _, root := range ancestors {
subdir := strings.TrimPrefix(prefix, root.key)
for _, rm := range root.roots {
if rm.fi.IsDir() {
fi, err := rm.fi.Meta().JoinStat(subdir)
if err == nil {
if err := collectDir(rm, fi); err != nil {
return nil, err
}
}
}
}
}

return fis, nil
}

Expand Down
56 changes: 53 additions & 3 deletions hugofs/rootmapping_fs_test.go
View file
Expand Up @@ -365,12 +365,18 @@ func TestRootMappingFsOs(t *testing.T) {

c.Assert(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755), qt.IsNil)

// https://github.com/gohugoio/hugo/issues/6854
mystaticDir := filepath.Join(d, "mystatic", "a", "b", "c")
c.Assert(fs.MkdirAll(mystaticDir, 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join(mystaticDir, "ms-1.txt"), []byte("some content"), 0755), qt.IsNil)

rfs, err := newRootMappingFsFromFromTo(
d,
fs,
"static/bf1", filepath.Join(d, "f1t"),
"static/cf2", filepath.Join(d, "f2t"),
"static/af3", filepath.Join(d, "f3t"),
"static", filepath.Join(d, "mystatic"),
"static/a/b/c", filepath.Join(d, "d1", "d2", "d3"),
"layouts", filepath.Join(d, "d1"),
)
Expand Down Expand Up @@ -400,13 +406,13 @@ func TestRootMappingFsOs(t *testing.T) {
}

c.Assert(getDirnames("static/a/b"), qt.DeepEquals, []string{"c"})
c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt"})
c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt", "ms-1.txt"})
c.Assert(getDirnames("static/a/b/c/d4"), qt.DeepEquals, []string{"d4-1", "d4-2", "d4-3", "d5"})

all, err := collectFilenames(rfs, "static", "static")
c.Assert(err, qt.IsNil)

c.Assert(all, qt.DeepEquals, []string{"a/b/c/f-1.txt", "a/b/c/f-2.txt", "a/b/c/f-3.txt", "cf2/myfile.txt"})
c.Assert(all, qt.DeepEquals, []string{"a/b/c/f-1.txt", "a/b/c/f-2.txt", "a/b/c/f-3.txt", "a/b/c/ms-1.txt", "cf2/myfile.txt"})

fis, err := collectFileinfos(rfs, "static", "static")
c.Assert(err, qt.IsNil)
Expand All @@ -423,7 +429,7 @@ func TestRootMappingFsOs(t *testing.T) {
sortFileInfos(fileInfos)
i := 0
for _, fi := range fileInfos {
if fi.IsDir() {
if fi.IsDir() || fi.Name() == "ms-1.txt" {
continue
}
i++
Expand All @@ -437,3 +443,47 @@ func TestRootMappingFsOs(t *testing.T) {
_, err = rfs.Stat(filepath.FromSlash("layouts/d2/d3"))
c.Assert(err, qt.IsNil)
}

func TestRootMappingFsOsBase(t *testing.T) {
c := qt.New(t)
fs := NewBaseFileDecorator(afero.NewOsFs())

d, clean, err := htesting.CreateTempDir(fs, "hugo-root-mapping-os-base")
c.Assert(err, qt.IsNil)
defer clean()

// Deep structure
deepDir := filepath.Join(d, "d1", "d2", "d3", "d4", "d5")
c.Assert(fs.MkdirAll(deepDir, 0755), qt.IsNil)
for i := 1; i <= 3; i++ {
c.Assert(fs.MkdirAll(filepath.Join(d, "d1", "d2", "d3", "d4", fmt.Sprintf("d4-%d", i)), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join(d, "d1", "d2", "d3", fmt.Sprintf("f-%d.txt", i)), []byte("some content"), 0755), qt.IsNil)
}

mystaticDir := filepath.Join(d, "mystatic", "a", "b", "c")
c.Assert(fs.MkdirAll(mystaticDir, 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join(mystaticDir, "ms-1.txt"), []byte("some content"), 0755), qt.IsNil)

bfs := afero.NewBasePathFs(fs, d)

rfs, err := newRootMappingFsFromFromTo(
"",
bfs,
"static", "mystatic",
"static/a/b/c", filepath.Join("d1", "d2", "d3"),
)

getDirnames := func(dirname string) []string {
dirname = filepath.FromSlash(dirname)
f, err := rfs.Open(dirname)
c.Assert(err, qt.IsNil)
defer f.Close()
dirnames, err := f.Readdirnames(-1)
c.Assert(err, qt.IsNil)
sort.Strings(dirnames)
return dirnames
}

c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt", "ms-1.txt"})

}

0 comments on commit b78576f

Please sign in to comment.