Cool
@@ -460,7 +463,7 @@ Cool
`)
- b.WithSourceFile(filepath.Join("assets", "scss", "styles2.scss"), `
+ b.WithSourceFile(filepath.Join("assets", "scss", "styles2.scss"), `
$color: #333;
body {
@@ -468,7 +471,7 @@ body {
}
`)
- b.WithSourceFile(filepath.Join("assets", "sass", "styles3.sass"), `
+ b.WithSourceFile(filepath.Join("assets", "sass", "styles3.sass"), `
$color: #333;
.content-navigation
@@ -476,10 +479,11 @@ $color: #333;
`)
- t.Log("Test", test.name)
- test.prepare(b)
- b.Build(BuildCfg{})
- test.verify(b)
+ test.prepare(b)
+ b.Build(BuildCfg{})
+ test.verify(b)
+
+ })
}
}
diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go
index 42eef61ae0a..24ef77503cb 100644
--- a/hugolib/shortcode_test.go
+++ b/hugolib/shortcode_test.go
@@ -26,10 +26,6 @@ import (
"strings"
"testing"
- "github.com/spf13/viper"
-
- "github.com/spf13/afero"
-
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/tpl"
@@ -54,12 +50,8 @@ title: "Title"
writeSource(t, fs, "content/simple.md", contentFile)
- h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg, WithTemplate: withTemplate})
-
- require.NoError(t, err)
- require.Len(t, h.Sites, 1)
-
- err = h.Build(BuildCfg{})
+ b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg, WithTemplate: withTemplate}).WithNothingAdded()
+ err := b.BuildE(BuildCfg{})
if err != nil && !expectError {
t.Fatalf("Shortcode rendered error %s.", err)
@@ -69,6 +61,9 @@ title: "Title"
t.Fatalf("No error from shortcode")
}
+ h := b.H
+ require.Len(t, h.Sites, 1)
+
require.Len(t, h.Sites[0].RegularPages(), 1)
output := strings.TrimSpace(content(h.Sites[0].RegularPages()[0]))
@@ -78,7 +73,7 @@ title: "Title"
expected = strings.TrimSpace(expected)
if output != expected {
- Fatalf(t, "Shortcode render didn't match. got \n%q but expected \n%q", output, expected)
+ t.Fatalf("Shortcode render didn't match. got \n%q but expected \n%q", output, expected)
}
}
@@ -341,7 +336,6 @@ func TestShortcodeWrappedInPIssue(t *testing.T) {
}
func TestExtractShortcodes(t *testing.T) {
- t.Parallel()
b := newTestSitesBuilder(t).WithSimpleConfigFile()
b.WithTemplates(
@@ -413,7 +407,10 @@ title: "Shortcodes Galore!"
{"inline", `{{< my.inline >}}Hi{{< /my.inline >}}`, regexpCheck("my.inline;inline:true;closing:true;inner:{Hi};")},
} {
+ test := test
+
t.Run(test.name, func(t *testing.T) {
+ t.Parallel()
assert := require.New(t)
counter := 0
@@ -437,7 +434,6 @@ title: "Shortcodes Galore!"
}
func TestShortcodesInSite(t *testing.T) {
- t.Parallel()
baseURL := "http://foo/bar"
tests := []struct {
@@ -577,7 +573,9 @@ title: "Foo"
s := buildSingleSite(t, deps.DepsCfg{WithTemplate: addTemplates, Fs: fs, Cfg: cfg}, BuildCfg{})
for i, test := range tests {
+ test := test
t.Run(fmt.Sprintf("test=%d;contentPath=%s", i, test.contentPath), func(t *testing.T) {
+ t.Parallel()
if strings.HasSuffix(test.contentPath, ".ad") && !helpers.HasAsciidoc() {
t.Skip("Skip Asciidoc test case as no Asciidoc present.")
} else if strings.HasSuffix(test.contentPath, ".rst") && !helpers.HasRst() {
@@ -632,9 +630,8 @@ outputs: ["CSV"]
CSV: {{< myShort >}}
`
- mf := afero.NewMemMapFs()
-
- th, h := newTestSitesFromConfig(t, mf, siteConfig,
+ b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig)
+ b.WithTemplates(
"layouts/_default/single.html", `Single HTML: {{ .Title }}|{{ .Content }}`,
"layouts/_default/single.json", `Single JSON: {{ .Title }}|{{ .Content }}`,
"layouts/_default/single.csv", `Single CSV: {{ .Title }}|{{ .Content }}`,
@@ -651,14 +648,13 @@ CSV: {{< myShort >}}
"layouts/shortcodes/myInner.html", `myInner:--{{- .Inner -}}--`,
)
- fs := th.Fs
-
- writeSource(t, fs, "content/_index.md", fmt.Sprintf(pageTemplate, "Home"))
- writeSource(t, fs, "content/sect/mypage.md", fmt.Sprintf(pageTemplate, "Single"))
- writeSource(t, fs, "content/sect/mycsvpage.md", fmt.Sprintf(pageTemplateCSVOnly, "Single CSV"))
+ b.WithContent("_index.md", fmt.Sprintf(pageTemplate, "Home"),
+ "sect/mypage.md", fmt.Sprintf(pageTemplate, "Single"),
+ "sect/mycsvpage.md", fmt.Sprintf(pageTemplateCSVOnly, "Single CSV"),
+ )
- err := h.Build(BuildCfg{})
- require.NoError(t, err)
+ b.Build(BuildCfg{})
+ h := b.H
require.Len(t, h.Sites, 1)
s := h.Sites[0]
@@ -666,7 +662,7 @@ CSV: {{< myShort >}}
require.NotNil(t, home)
require.Len(t, home.OutputFormats(), 3)
- th.assertFileContent("public/index.html",
+ b.AssertFileContent("public/index.html",
"Home HTML",
"ShortHTML",
"ShortNoExt",
@@ -674,7 +670,7 @@ CSV: {{< myShort >}}
"myInner:--ShortHTML--",
)
- th.assertFileContent("public/amp/index.html",
+ b.AssertFileContent("public/amp/index.html",
"Home AMP",
"ShortAMP",
"ShortNoExt",
@@ -682,7 +678,7 @@ CSV: {{< myShort >}}
"myInner:--ShortAMP--",
)
- th.assertFileContent("public/index.ics",
+ b.AssertFileContent("public/index.ics",
"Home Calendar",
"ShortCalendar",
"ShortNoExt",
@@ -690,7 +686,7 @@ CSV: {{< myShort >}}
"myInner:--ShortCalendar--",
)
- th.assertFileContent("public/sect/mypage/index.html",
+ b.AssertFileContent("public/sect/mypage/index.html",
"Single HTML",
"ShortHTML",
"ShortNoExt",
@@ -698,7 +694,7 @@ CSV: {{< myShort >}}
"myInner:--ShortHTML--",
)
- th.assertFileContent("public/sect/mypage/index.json",
+ b.AssertFileContent("public/sect/mypage/index.json",
"Single JSON",
"ShortJSON",
"ShortNoExt",
@@ -706,7 +702,7 @@ CSV: {{< myShort >}}
"myInner:--ShortJSON--",
)
- th.assertFileContent("public/amp/sect/mypage/index.html",
+ b.AssertFileContent("public/amp/sect/mypage/index.html",
// No special AMP template
"Single HTML",
"ShortAMP",
@@ -715,7 +711,7 @@ CSV: {{< myShort >}}
"myInner:--ShortAMP--",
)
- th.assertFileContent("public/sect/mycsvpage/index.csv",
+ b.AssertFileContent("public/sect/mycsvpage/index.csv",
"Single CSV",
"ShortCSV",
)
@@ -864,10 +860,6 @@ weight: %d
---
C-%s`
- v := viper.New()
-
- v.Set("timeout", 500)
-
templates = append(templates, []string{"shortcodes/c.html", contentShortcode}...)
templates = append(templates, []string{"_default/single.html", "Single Content: {{ .Content }}"}...)
templates = append(templates, []string{"_default/list.html", "List Content: {{ .Content }}"}...)
@@ -884,21 +876,21 @@ C-%s`
builder := newTestSitesBuilder(t).WithDefaultMultiSiteConfig()
- builder.WithViper(v).WithContent(content...).WithTemplates(templates...).CreateSites().Build(BuildCfg{})
+ builder.WithContent(content...).WithTemplates(templates...).CreateSites().Build(BuildCfg{})
s := builder.H.Sites[0]
assert.Equal(3, len(s.RegularPages()))
- builder.AssertFileContent("public/section1/index.html",
+ builder.AssertFileContent("public/en/section1/index.html",
"List Content: Logo:P1:|P2:logo.png/PNG logo|:P1: P1:|P2:docs1p1/
C-s1p1
\n|",
"BP1:P1:|P2:docbp1/C-bp1
",
)
- builder.AssertFileContent("public/b1/index.html",
+ builder.AssertFileContent("public/en/b1/index.html",
"Single Content: Logo:P1:|P2:logo.png/PNG logo|:P1: P1:|P2:docs1p1/
C-s1p1
\n|",
"P2:docbp1/C-bp1
",
)
- builder.AssertFileContent("public/section2/s2p1/index.html",
+ builder.AssertFileContent("public/en/section2/s2p1/index.html",
"Single Content: Logo:P1:|P2:logo.png/PNG logo|:P1: P1:|P2:docs1p1/
C-s1p1
\n|",
"P2:docbp1/C-bp1
",
)
@@ -1062,8 +1054,10 @@ String: {{ . | safeHTML }}
func TestInlineShortcodes(t *testing.T) {
for _, enableInlineShortcodes := range []bool{true, false} {
+ enableInlineShortcodes := enableInlineShortcodes
t.Run(fmt.Sprintf("enableInlineShortcodes=%t", enableInlineShortcodes),
func(t *testing.T) {
+ t.Parallel()
conf := fmt.Sprintf(`
baseURL = "https://example.com"
enableInlineShortcodes = %t
diff --git a/hugolib/site.go b/hugolib/site.go
index b1441ca8a13..fa6598273b2 100644
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -14,7 +14,6 @@
package hugolib
import (
- "context"
"fmt"
"html/template"
"io"
@@ -29,6 +28,8 @@ import (
"strings"
"time"
+ "github.com/gohugoio/hugo/hugofs/files"
+
"github.com/gohugoio/hugo/common/maps"
"github.com/pkg/errors"
@@ -45,7 +46,6 @@ import (
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/lazy"
- "golang.org/x/sync/errgroup"
"github.com/gohugoio/hugo/media"
@@ -1028,7 +1028,8 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) {
removed = true
}
}
- if removed && IsContentFile(ev.Name) {
+
+ if removed && files.IsContentFile(ev.Name) {
h.removePageByFilename(ev.Name)
}
@@ -1078,10 +1079,12 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) {
func (s *Site) process(config BuildCfg) (err error) {
if err = s.initialize(); err != nil {
+ err = errors.Wrap(err, "initialize")
return
}
- if err := s.readAndProcessContent(); err != nil {
- return err
+ if err = s.readAndProcessContent(); err != nil {
+ err = errors.Wrap(err, "readAndProcessContent")
+ return
}
return err
@@ -1304,93 +1307,14 @@ func (s *Site) isContentDirEvent(e fsnotify.Event) bool {
return s.BaseFs.IsContent(e.Name)
}
-type contentCaptureResultHandler struct {
- defaultContentProcessor *siteContentProcessor
- contentProcessors map[string]*siteContentProcessor
-}
-
-func (c *contentCaptureResultHandler) getContentProcessor(lang string) *siteContentProcessor {
- proc, found := c.contentProcessors[lang]
- if found {
- return proc
- }
- return c.defaultContentProcessor
-}
-
-func (c *contentCaptureResultHandler) handleSingles(fis ...*fileInfo) {
- for _, fi := range fis {
- proc := c.getContentProcessor(fi.Lang())
- proc.processSingle(fi)
- }
-}
-func (c *contentCaptureResultHandler) handleBundles(d *bundleDirs) {
- for _, b := range d.bundles {
- proc := c.getContentProcessor(b.fi.Lang())
- proc.processBundle(b)
- }
-}
-
-func (c *contentCaptureResultHandler) handleCopyFile(f pathLangFile) {
- proc := c.getContentProcessor(f.Lang())
- proc.processAsset(f)
-}
-
func (s *Site) readAndProcessContent(filenames ...string) error {
-
- ctx := context.Background()
- g, ctx := errgroup.WithContext(ctx)
-
- defaultContentLanguage := s.SourceSpec.DefaultContentLanguage
-
- contentProcessors := make(map[string]*siteContentProcessor)
- var defaultContentProcessor *siteContentProcessor
- sites := s.h.langSite()
- for k, v := range sites {
- if v.language.Disabled {
- continue
- }
- proc := newSiteContentProcessor(ctx, len(filenames) > 0, v)
- contentProcessors[k] = proc
- if k == defaultContentLanguage {
- defaultContentProcessor = proc
- }
- g.Go(func() error {
- return proc.process(ctx)
- })
- }
-
- var (
- handler captureResultHandler
- bundleMap *contentChangeMap
- )
-
- mainHandler := &contentCaptureResultHandler{contentProcessors: contentProcessors, defaultContentProcessor: defaultContentProcessor}
-
sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.Content.Fs)
- if s.running() {
- // Need to track changes.
- bundleMap = s.h.ContentChanges
- handler = &captureResultHandlerChain{handlers: []captureBundlesHandler{mainHandler, bundleMap}}
-
- } else {
- handler = mainHandler
- }
-
- c := newCapturer(s.Log, sourceSpec, handler, bundleMap, filenames...)
+ proc := newPagesProcessor(s.h, sourceSpec, len(filenames) > 0)
- err1 := c.capture()
+ c := newPagesCollector(sourceSpec, s.Log, s.h.ContentChanges, proc, filenames...)
- for _, proc := range contentProcessors {
- proc.closeInput()
- }
-
- err2 := g.Wait()
-
- if err1 != nil {
- return err1
- }
- return err2
+ return c.Collect()
}
func (s *Site) getMenusFromConfig() navigation.Menus {
@@ -1831,8 +1755,8 @@ func (s *Site) kindFromFileInfoOrSections(fi *fileInfo, sections []string) strin
}
func (s *Site) kindFromSections(sections []string) string {
- if len(sections) == 0 || len(s.siteCfg.taxonomiesConfig) == 0 {
- return page.KindSection
+ if len(sections) == 0 {
+ return page.KindHome
}
sectionPath := path.Join(sections...)
diff --git a/hugolib/site_output_test.go b/hugolib/site_output_test.go
index 71b87b63692..1536817b091 100644
--- a/hugolib/site_output_test.go
+++ b/hugolib/site_output_test.go
@@ -32,14 +32,15 @@ import (
func TestSiteWithPageOutputs(t *testing.T) {
for _, outputs := range [][]string{{"html", "json", "calendar"}, {"json"}} {
+ outputs := outputs
t.Run(fmt.Sprintf("%v", outputs), func(t *testing.T) {
+ t.Parallel()
doTestSiteWithPageOutputs(t, outputs)
})
}
}
func doTestSiteWithPageOutputs(t *testing.T, outputs []string) {
- t.Parallel()
outputsStr := strings.Replace(fmt.Sprintf("%q", outputs), " ", ", ", -1)
@@ -84,19 +85,16 @@ outputs: %s
`
- mf := afero.NewMemMapFs()
-
- writeToFs(t, mf, "i18n/en.toml", `
+ b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig)
+ b.WithI18n("en.toml", `
[elbow]
other = "Elbow"
-`)
- writeToFs(t, mf, "i18n/nn.toml", `
+`, "nn.toml", `
[elbow]
other = "Olboge"
`)
- th, h := newTestSitesFromConfig(t, mf, siteConfig,
-
+ b.WithTemplates(
// Case issue partials #3333
"layouts/partials/GoHugo.html", `Go Hugo Partial`,
"layouts/_default/baseof.json", `START JSON:{{block "main" .}}default content{{ end }}:END JSON`,
@@ -133,23 +131,17 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P
`,
"layouts/_default/single.html", `{{ define "main" }}{{ .Content }}{{ end }}`,
)
- require.Len(t, h.Sites, 2)
-
- fs := th.Fs
- writeSource(t, fs, "content/_index.md", fmt.Sprintf(pageTemplate, "JSON Home", outputsStr))
- writeSource(t, fs, "content/_index.nn.md", fmt.Sprintf(pageTemplate, "JSON Nynorsk Heim", outputsStr))
+ b.WithContent("_index.md", fmt.Sprintf(pageTemplate, "JSON Home", outputsStr))
+ b.WithContent("_index.nn.md", fmt.Sprintf(pageTemplate, "JSON Nynorsk Heim", outputsStr))
for i := 1; i <= 10; i++ {
- writeSource(t, fs, fmt.Sprintf("content/p%d.md", i), fmt.Sprintf(pageTemplate, fmt.Sprintf("Page %d", i), outputsStr))
-
+ b.WithContent(fmt.Sprintf("p%d.md", i), fmt.Sprintf(pageTemplate, fmt.Sprintf("Page %d", i), outputsStr))
}
- err := h.Build(BuildCfg{})
-
- require.NoError(t, err)
+ b.Build(BuildCfg{})
- s := h.Sites[0]
+ s := b.H.Sites[0]
require.Equal(t, "en", s.language.Lang)
home := s.getPage(page.KindHome)
@@ -163,13 +155,13 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P
// There is currently always a JSON output to make it simpler ...
altFormats := lenOut - 1
hasHTML := helpers.InStringArray(outputs, "html")
- th.assertFileContent("public/index.json",
+ b.AssertFileContent("public/index.json",
"List JSON",
fmt.Sprintf("Alt formats: %d", altFormats),
)
if hasHTML {
- th.assertFileContent("public/index.json",
+ b.AssertFileContent("public/index.json",
"Alt Output: HTML",
"Output/Rel: JSON/alternate|",
"Output/Rel: HTML/canonical|",
@@ -178,7 +170,7 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P
"OtherShort: Hi!
",
)
- th.assertFileContent("public/index.html",
+ b.AssertFileContent("public/index.html",
// The HTML entity is a deliberate part of this test: The HTML templates are
// parsed with html/template.
`List HTML|JSON Home|`,
@@ -187,21 +179,22 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P
"OtherShort: Hi!
",
"Len Pages: home 10",
)
- th.assertFileContent("public/page/2/index.html", "Page Number: 2")
- th.assertFileNotExist("public/page/2/index.json")
+ assert := require.New(t)
+ b.AssertFileContent("public/page/2/index.html", "Page Number: 2")
+ assert.False(b.CheckExists("public/page/2/index.json"))
- th.assertFileContent("public/nn/index.html",
+ b.AssertFileContent("public/nn/index.html",
"List HTML|JSON Nynorsk Heim|",
"nn: Olboge")
} else {
- th.assertFileContent("public/index.json",
+ b.AssertFileContent("public/index.json",
"Output/Rel: JSON/canonical|",
// JSON is plain text, so no need to safeHTML this and that
``,
"ShortJSON",
"OtherShort: Hi!
",
)
- th.assertFileContent("public/nn/index.json",
+ b.AssertFileContent("public/nn/index.json",
"List JSON|JSON Nynorsk Heim|",
"nn: Olboge",
"ShortJSON",
diff --git a/hugolib/site_sections_test.go b/hugolib/site_sections_test.go
index 199947c3131..d4aa9d354b9 100644
--- a/hugolib/site_sections_test.go
+++ b/hugolib/site_sections_test.go
@@ -25,7 +25,6 @@ import (
)
func TestNestedSections(t *testing.T) {
- t.Parallel()
var (
assert = require.New(t)
@@ -294,7 +293,9 @@ PAG|{{ .Title }}|{{ $sect.InSection . }}
home := s.getPage(page.KindHome)
for _, test := range tests {
+ test := test
t.Run(fmt.Sprintf("sections %s", test.sections), func(t *testing.T) {
+ t.Parallel()
assert := require.New(t)
sections := strings.Split(test.sections, ",")
p := s.getPage(page.KindSection, sections...)
diff --git a/hugolib/site_stats_test.go b/hugolib/site_stats_test.go
index c722037b4eb..bbefc95774f 100644
--- a/hugolib/site_stats_test.go
+++ b/hugolib/site_stats_test.go
@@ -20,7 +20,6 @@ import (
"testing"
"github.com/gohugoio/hugo/helpers"
- "github.com/spf13/afero"
"github.com/stretchr/testify/require"
)
@@ -60,30 +59,28 @@ aliases: [/Ali%d]
# Doc
`
- th, h := newTestSitesFromConfig(t, afero.NewMemMapFs(), siteConfig,
- "layouts/_default/single.html", "Single|{{ .Title }}|{{ .Content }}",
- "layouts/_default/list.html", `List|{{ .Title }}|Pages: {{ .Paginator.TotalPages }}|{{ .Content }}`,
- "layouts/_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}",
- )
- require.Len(t, h.Sites, 2)
+ b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig)
- fs := th.Fs
+ b.WithTemplates(
+ "_default/single.html", "Single|{{ .Title }}|{{ .Content }}",
+ "_default/list.html", `List|{{ .Title }}|Pages: {{ .Paginator.TotalPages }}|{{ .Content }}`,
+ "_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}",
+ )
for i := 0; i < 2; i++ {
for j := 0; j < 2; j++ {
pageID := i + j + 1
- writeSource(t, fs, fmt.Sprintf("content/sect/p%d.md", pageID),
+ b.WithContent(fmt.Sprintf("content/sect/p%d.md", pageID),
fmt.Sprintf(pageTemplate, pageID, fmt.Sprintf("- tag%d", j), fmt.Sprintf("- category%d", j), pageID))
}
}
for i := 0; i < 5; i++ {
- writeSource(t, fs, fmt.Sprintf("content/assets/image%d.png", i+1), "image")
+ b.WithContent(fmt.Sprintf("assets/image%d.png", i+1), "image")
}
- err := h.Build(BuildCfg{})
-
- assert.NoError(err)
+ b.Build(BuildCfg{})
+ h := b.H
stats := []*helpers.ProcessingStats{
h.Sites[0].PathSpec.ProcessingStats,
diff --git a/hugolib/site_test.go b/hugolib/site_test.go
index 5912abbc9c0..bbf101fc406 100644
--- a/hugolib/site_test.go
+++ b/hugolib/site_test.go
@@ -362,11 +362,14 @@ func TestShouldNotWriteZeroLengthFilesToDestination(t *testing.T) {
// Issue #1176
func TestSectionNaming(t *testing.T) {
- t.Parallel()
for _, canonify := range []bool{true, false} {
for _, uglify := range []bool{true, false} {
for _, pluralize := range []bool{true, false} {
+ canonify := canonify
+ uglify := uglify
+ pluralize := pluralize
t.Run(fmt.Sprintf("canonify=%t,uglify=%t,pluralize=%t", canonify, uglify, pluralize), func(t *testing.T) {
+ t.Parallel()
doTestSectionNaming(t, canonify, uglify, pluralize)
})
}
diff --git a/hugolib/taxonomy_test.go b/hugolib/taxonomy_test.go
index f4902ae8d65..2edc36d63e1 100644
--- a/hugolib/taxonomy_test.go
+++ b/hugolib/taxonomy_test.go
@@ -64,14 +64,15 @@ YAML frontmatter with tags and categories taxonomy.`
//
func TestTaxonomiesWithAndWithoutContentFile(t *testing.T) {
for _, uglyURLs := range []bool{false, true} {
+ uglyURLs := uglyURLs
t.Run(fmt.Sprintf("uglyURLs=%t", uglyURLs), func(t *testing.T) {
+ t.Parallel()
doTestTaxonomiesWithAndWithoutContentFile(t, uglyURLs)
})
}
}
func doTestTaxonomiesWithAndWithoutContentFile(t *testing.T, uglyURLs bool) {
- t.Parallel()
siteConfig := `
baseURL = "http://example.com/blog"
@@ -104,25 +105,20 @@ permalinkeds:
siteConfig = fmt.Sprintf(siteConfig, uglyURLs)
- th, h := newTestSitesFromConfigWithDefaultTemplates(t, siteConfig)
- require.Len(t, h.Sites, 1)
-
- fs := th.Fs
-
- writeSource(t, fs, "content/p1.md", fmt.Sprintf(pageTemplate, "t1/c1", "- Tag1", "- cAt1", "- o1", "- Pl1"))
- writeSource(t, fs, "content/p2.md", fmt.Sprintf(pageTemplate, "t2/c1", "- tag2", "- cAt1", "- o1", "- Pl1"))
- writeSource(t, fs, "content/p3.md", fmt.Sprintf(pageTemplate, "t2/c12", "- tag2", "- cat2", "- o1", "- Pl1"))
- writeSource(t, fs, "content/p4.md", fmt.Sprintf(pageTemplate, "Hello World", "", "", "- \"Hello Hugo world\"", "- Pl1"))
+ b := newTestSitesBuilder(t).WithConfigFile("toml", siteConfig)
- writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10)
- writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/Tag1/_index.md", 10)
+ b.WithContent(
+ "p1.md", fmt.Sprintf(pageTemplate, "t1/c1", "- Tag1", "- cAt1", "- o1", "- Pl1"),
+ "p2.md", fmt.Sprintf(pageTemplate, "t2/c1", "- tag2", "- cAt1", "- o1", "- Pl1"),
+ "p3.md", fmt.Sprintf(pageTemplate, "t2/c12", "- tag2", "- cat2", "- o1", "- Pl1"),
+ "p4.md", fmt.Sprintf(pageTemplate, "Hello World", "", "", "- \"Hello Hugo world\"", "- Pl1"),
+ "categories/_index.md", newTestPage("Category Terms", "2017-01-01", 10),
+ "tags/Tag1/_index.md", newTestPage("Tag1 List", "2017-01-01", 10),
+ // https://github.com/gohugoio/hugo/issues/5847
+ "/tags/not-used/_index.md", newTestPage("Unused Tag List", "2018-01-01", 10),
+ )
- // https://github.com/gohugoio/hugo/issues/5847
- writeNewContentFile(t, fs.Source, "Unused Tag List", "2018-01-01", "content/tags/not-used/_index.md", 10)
-
- err := h.Build(BuildCfg{})
-
- require.NoError(t, err)
+ b.Build(BuildCfg{})
// So what we have now is:
// 1. categories with terms content page, but no content page for the only c1 category
@@ -138,26 +134,26 @@ permalinkeds:
}
// 1.
- th.assertFileContent(pathFunc("public/categories/cat1/index.html"), "List", "cAt1")
- th.assertFileContent(pathFunc("public/categories/index.html"), "Terms List", "Category Terms")
+ b.AssertFileContent(pathFunc("public/categories/cat1/index.html"), "List", "cAt1")
+ b.AssertFileContent(pathFunc("public/categories/index.html"), "Taxonomy Term Page", "Category Terms")
// 2.
- th.assertFileContent(pathFunc("public/tags/tag2/index.html"), "List", "tag2")
- th.assertFileContent(pathFunc("public/tags/tag1/index.html"), "List", "Tag1")
- th.assertFileContent(pathFunc("public/tags/index.html"), "Terms List", "Tags")
+ b.AssertFileContent(pathFunc("public/tags/tag2/index.html"), "List", "tag2")
+ b.AssertFileContent(pathFunc("public/tags/tag1/index.html"), "List", "Tag1")
+ b.AssertFileContent(pathFunc("public/tags/index.html"), "Taxonomy Term Page", "Tags")
// 3.
- th.assertFileContent(pathFunc("public/others/o1/index.html"), "List", "o1")
- th.assertFileContent(pathFunc("public/others/index.html"), "Terms List", "Others")
+ b.AssertFileContent(pathFunc("public/others/o1/index.html"), "List", "o1")
+ b.AssertFileContent(pathFunc("public/others/index.html"), "Taxonomy Term Page", "Others")
// 4.
- th.assertFileContent(pathFunc("public/perma/pl1/index.html"), "List", "Pl1")
+ b.AssertFileContent(pathFunc("public/perma/pl1/index.html"), "List", "Pl1")
// This looks kind of funky, but the taxonomy terms do not have a permalinks definition,
// for good reasons.
- th.assertFileContent(pathFunc("public/permalinkeds/index.html"), "Terms List", "Permalinkeds")
+ b.AssertFileContent(pathFunc("public/permalinkeds/index.html"), "Taxonomy Term Page", "Permalinkeds")
- s := h.Sites[0]
+ s := b.H.Sites[0]
// Make sure that each page.KindTaxonomyTerm page has an appropriate number
// of page.KindTaxonomy pages in its Pages slice.
@@ -204,7 +200,7 @@ permalinkeds:
require.Equal(t, "Hello Hugo world", helloWorld.Title())
// Issue #2977
- th.assertFileContent(pathFunc("public/empties/index.html"), "Terms List", "Empties")
+ b.AssertFileContent(pathFunc("public/empties/index.html"), "Taxonomy Term Page", "Empties")
}
diff --git a/hugolib/template_engines_test.go b/hugolib/template_engines_test.go
index 6a046c9f59a..ec229a29941 100644
--- a/hugolib/template_engines_test.go
+++ b/hugolib/template_engines_test.go
@@ -24,7 +24,6 @@ import (
)
func TestAllTemplateEngines(t *testing.T) {
- t.Parallel()
noOp := func(s string) string {
return s
}
@@ -48,8 +47,10 @@ func TestAllTemplateEngines(t *testing.T) {
{"html", noOp},
{"ace", noOp},
} {
+ config := config
t.Run(config.suffix,
func(t *testing.T) {
+ t.Parallel()
doTestTemplateEngine(t, config.suffix, config.templateFixer)
})
}
diff --git a/hugolib/template_test.go b/hugolib/template_test.go
index 3ec81323b4b..6ed9643c7fd 100644
--- a/hugolib/template_test.go
+++ b/hugolib/template_test.go
@@ -25,7 +25,6 @@ import (
)
func TestTemplateLookupOrder(t *testing.T) {
- t.Parallel()
var (
fs *hugofs.Fs
cfg *viper.Viper
@@ -193,22 +192,26 @@ func TestTemplateLookupOrder(t *testing.T) {
},
} {
- cfg, fs = newTestCfg()
- th = testHelper{cfg, fs, t}
+ this := this
+ t.Run(this.name, func(t *testing.T) {
+ // TODO(bep) there are some function vars need to pull down here to enable => t.Parallel()
+ cfg, fs = newTestCfg()
+ th = testHelper{cfg, fs, t}
- for i := 1; i <= 3; i++ {
- writeSource(t, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", i)), `---
+ for i := 1; i <= 3; i++ {
+ writeSource(t, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", i)), `---
title: Template test
---
Some content
`)
- }
+ }
- this.setup(t)
+ this.setup(t)
- buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{})
- t.Log(this.name)
- this.assert(t)
+ buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{})
+ //helpers.PrintFs(s.BaseFs.Layouts.Fs, "", os.Stdout)
+ this.assert(t)
+ })
}
}
diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go
index 8c72e10d0c1..dbce1dc48be 100644
--- a/hugolib/testhelpers_test.go
+++ b/hugolib/testhelpers_test.go
@@ -2,13 +2,17 @@ package hugolib
import (
"io"
- "io/ioutil"
"path/filepath"
"runtime"
"strconv"
"testing"
"unicode/utf8"
+ "github.com/gohugoio/hugo/parser/metadecoders"
+
+ "github.com/gohugoio/hugo/parser"
+ "github.com/pkg/errors"
+
"bytes"
"fmt"
"regexp"
@@ -39,9 +43,10 @@ import (
)
type sitesBuilder struct {
- Cfg config.Provider
- Fs *hugofs.Fs
- T testing.TB
+ Cfg config.Provider
+ Fs *hugofs.Fs
+ T testing.TB
+ depsCfg deps.DepsCfg
*require.Assertions
@@ -60,13 +65,16 @@ type sitesBuilder struct {
theme string
// Default toml
- configFormat string
+ configFormat string
+ configFileSet bool
+ viperSet bool
// Default is empty.
// TODO(bep) revisit this and consider always setting it to something.
// Consider this in relation to using the BaseFs.PublishFs to all publishing.
workingDir string
+ addNothing bool
// Base data/content
contentFilePairs []string
templateFilePairs []string
@@ -94,18 +102,22 @@ func newTestSitesBuilder(t testing.TB) *sitesBuilder {
return &sitesBuilder{T: t, Assertions: require.New(t), Fs: fs, configFormat: "toml", dumper: litterOptions}
}
-func createTempDir(prefix string) (string, func(), error) {
- workDir, err := ioutil.TempDir("", prefix)
- if err != nil {
- return "", nil, err
- }
+func newTestSitesBuilderFromDepsCfg(t testing.TB, d deps.DepsCfg) *sitesBuilder {
+ assert := require.New(t)
- if runtime.GOOS == "darwin" && !strings.HasPrefix(workDir, "/private") {
- // To get the entry folder in line with the rest. This its a little bit
- // mysterious, but so be it.
- workDir = "/private" + workDir
+ litterOptions := litter.Options{
+ HidePrivateFields: true,
+ StripPackageNames: true,
+ Separator: " ",
}
- return workDir, func() { os.RemoveAll(workDir) }, nil
+
+ b := &sitesBuilder{T: t, Assertions: assert, depsCfg: d, Fs: d.Fs, dumper: litterOptions}
+ workingDir := d.Cfg.GetString("workingDir")
+
+ b.WithWorkingDir(workingDir)
+
+ return b.WithViper(d.Cfg.(*viper.Viper))
+
}
func (s *sitesBuilder) Running() *sitesBuilder {
@@ -113,17 +125,24 @@ func (s *sitesBuilder) Running() *sitesBuilder {
return s
}
+func (s *sitesBuilder) WithNothingAdded() *sitesBuilder {
+ s.addNothing = true
+ return s
+}
+
func (s *sitesBuilder) WithLogger(logger *loggers.Logger) *sitesBuilder {
s.logger = logger
return s
}
func (s *sitesBuilder) WithWorkingDir(dir string) *sitesBuilder {
- s.workingDir = dir
+ s.workingDir = filepath.FromSlash(dir)
return s
}
func (s *sitesBuilder) WithConfigTemplate(data interface{}, format, configTemplate string) *sitesBuilder {
+ s.T.Helper()
+
if format == "" {
format = "toml"
}
@@ -138,32 +157,59 @@ func (s *sitesBuilder) WithConfigTemplate(data interface{}, format, configTempla
}
func (s *sitesBuilder) WithViper(v *viper.Viper) *sitesBuilder {
- loadDefaultSettingsFor(v)
- s.Cfg = v
+ s.T.Helper()
+ if s.configFileSet {
+ s.T.Fatal("WithViper: use Viper or config.toml, not both")
+ }
+ defer func() {
+ s.viperSet = true
+ }()
- return s
+ // Write to a config file to make sure the tests follow the same code path.
+ var buff bytes.Buffer
+ m := v.AllSettings()
+ s.Assertions.NoError(parser.InterfaceToConfig(m, metadecoders.TOML, &buff))
+ return s.WithConfigFile("toml", buff.String())
}
func (s *sitesBuilder) WithConfigFile(format, conf string) *sitesBuilder {
- writeSource(s.T, s.Fs, "config."+format, conf)
+ s.T.Helper()
+ if s.viperSet {
+ s.T.Fatal("WithConfigFile: use Viper or config.toml, not both")
+ }
+ s.configFileSet = true
+ filename := s.absFilename("config." + format)
+ writeSource(s.T, s.Fs, filename, conf)
s.configFormat = format
return s
}
func (s *sitesBuilder) WithThemeConfigFile(format, conf string) *sitesBuilder {
+ s.T.Helper()
if s.theme == "" {
s.theme = "test-theme"
}
filename := filepath.Join("themes", s.theme, "config."+format)
- writeSource(s.T, s.Fs, filename, conf)
+ writeSource(s.T, s.Fs, s.absFilename(filename), conf)
return s
}
-func (s *sitesBuilder) WithSourceFile(filename, content string) *sitesBuilder {
- writeSource(s.T, s.Fs, filepath.FromSlash(filename), content)
+func (s *sitesBuilder) WithSourceFile(filenameContent ...string) *sitesBuilder {
+ s.T.Helper()
+ for i := 0; i < len(filenameContent); i += 2 {
+ writeSource(s.T, s.Fs, s.absFilename(filenameContent[i]), filenameContent[i+1])
+ }
return s
}
+func (s *sitesBuilder) absFilename(filename string) string {
+ filename = filepath.FromSlash(filename)
+ if s.workingDir != "" && !strings.HasPrefix(filename, s.workingDir) {
+ filename = filepath.Join(s.workingDir, filename)
+ }
+ return filename
+}
+
const commonConfigSections = `
[services]
@@ -191,10 +237,12 @@ privacyEnhanced = true
`
func (s *sitesBuilder) WithSimpleConfigFile() *sitesBuilder {
+ s.T.Helper()
return s.WithSimpleConfigFileAndBaseURL("http://example.com/")
}
func (s *sitesBuilder) WithSimpleConfigFileAndBaseURL(baseURL string) *sitesBuilder {
+ s.T.Helper()
config := fmt.Sprintf("baseURL = %q", baseURL)
config = config + commonConfigSections
@@ -323,7 +371,7 @@ func (s *sitesBuilder) EditFiles(filenameContent ...string) *sitesBuilder {
for i := 0; i < len(filenameContent); i += 2 {
filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1]
changedFiles = append(changedFiles, filename)
- writeSource(s.T, s.Fs, filename, content)
+ writeSource(s.T, s.Fs, s.absFilename(filename), content)
}
s.changedFiles = changedFiles
@@ -354,6 +402,7 @@ func (s *sitesBuilder) writeFilePairs(folder string, filenameContent []string) *
func (s *sitesBuilder) CreateSites() *sitesBuilder {
if err := s.CreateSitesE(); err != nil {
+ herrors.PrintStackTrace(err)
s.Fatalf("Failed to create sites: %s", err)
}
@@ -361,34 +410,68 @@ func (s *sitesBuilder) CreateSites() *sitesBuilder {
}
func (s *sitesBuilder) LoadConfig() error {
- cfg, _, err := LoadConfig(ConfigSourceDescriptor{Fs: s.Fs.Source, Filename: "config." + s.configFormat})
+ if !s.configFileSet {
+ s.WithSimpleConfigFile()
+ }
+
+ cfg, _, err := LoadConfig(ConfigSourceDescriptor{
+ WorkingDir: s.workingDir,
+ Fs: s.Fs.Source,
+ Filename: "config." + s.configFormat}, func(cfg config.Provider) error {
+
+ return nil
+ })
+
if err != nil {
return err
}
+
s.Cfg = cfg
+
return nil
}
func (s *sitesBuilder) CreateSitesE() error {
- s.addDefaults()
- s.writeFilePairs("content", s.contentFilePairs)
- s.writeFilePairs("content", s.contentFilePairsAdded)
- s.writeFilePairs("layouts", s.templateFilePairs)
- s.writeFilePairs("layouts", s.templateFilePairsAdded)
- s.writeFilePairs("data", s.dataFilePairs)
- s.writeFilePairs("data", s.dataFilePairsAdded)
- s.writeFilePairs("i18n", s.i18nFilePairs)
- s.writeFilePairs("i18n", s.i18nFilePairsAdded)
-
- if s.Cfg == nil {
- if err := s.LoadConfig(); err != nil {
- return err
+ if !s.addNothing {
+ if _, ok := s.Fs.Source.(*afero.OsFs); ok {
+ for _, dir := range []string{
+ "content/sect",
+ "layouts/_default",
+ "layouts/partials",
+ "layouts/shortcodes",
+ "data",
+ "i18n",
+ } {
+ if err := os.MkdirAll(filepath.Join(s.workingDir, dir), 0777); err != nil {
+ return errors.Wrapf(err, "failed to create %q", dir)
+ }
+ }
}
+
+ s.addDefaults()
+ s.writeFilePairs("content", s.contentFilePairs)
+ s.writeFilePairs("content", s.contentFilePairsAdded)
+ s.writeFilePairs("layouts", s.templateFilePairs)
+ s.writeFilePairs("layouts", s.templateFilePairsAdded)
+ s.writeFilePairs("data", s.dataFilePairs)
+ s.writeFilePairs("data", s.dataFilePairsAdded)
+ s.writeFilePairs("i18n", s.i18nFilePairs)
+ s.writeFilePairs("i18n", s.i18nFilePairsAdded)
}
- sites, err := NewHugoSites(deps.DepsCfg{Fs: s.Fs, Cfg: s.Cfg, Logger: s.logger, Running: s.running})
+ if err := s.LoadConfig(); err != nil {
+ return errors.Wrap(err, "failed to load config")
+ }
+
+ depsCfg := s.depsCfg
+ depsCfg.Fs = s.Fs
+ depsCfg.Cfg = s.Cfg
+ depsCfg.Logger = s.logger
+ depsCfg.Running = s.running
+
+ sites, err := NewHugoSites(depsCfg)
if err != nil {
- return err
+ return errors.Wrap(err, "failed to create sites")
}
s.H = sites
@@ -404,10 +487,12 @@ func (s *sitesBuilder) BuildE(cfg BuildCfg) error {
}
func (s *sitesBuilder) Build(cfg BuildCfg) *sitesBuilder {
+ s.T.Helper()
return s.build(cfg, false)
}
func (s *sitesBuilder) BuildFail(cfg BuildCfg) *sitesBuilder {
+ s.T.Helper()
return s.build(cfg, true)
}
@@ -528,14 +613,8 @@ hello:
}
func (s *sitesBuilder) Fatalf(format string, args ...interface{}) {
- Fatalf(s.T, format, args...)
-}
-
-func Fatalf(t testing.TB, format string, args ...interface{}) {
- trace := stackTrace()
- format = format + "\n%s"
- args = append(args, trace)
- t.Fatalf(format, args...)
+ s.T.Helper()
+ s.T.Fatalf(format, args...)
}
func stackTrace() string {
@@ -554,6 +633,11 @@ func (s *sitesBuilder) AssertHome(matches ...string) {
}
func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) {
+ s.T.Helper()
+ filename = filepath.FromSlash(filename)
+ if !strings.HasPrefix(filename, s.workingDir) {
+ filename = filepath.Join(s.workingDir, filename)
+ }
content := s.FileContent(filename)
for _, match := range matches {
if !strings.Contains(content, match) {
@@ -567,6 +651,7 @@ func (s *sitesBuilder) FileContent(filename string) string {
}
func (s *sitesBuilder) AssertObject(expected string, object interface{}) {
+ s.T.Helper()
got := s.dumper.Sdump(object)
expected = strings.TrimSpace(expected)
@@ -633,17 +718,41 @@ func (th testHelper) replaceDefaultContentLanguageValue(value string) string {
return value
}
-func newTestCfg() (*viper.Viper, *hugofs.Fs) {
+func loadTestConfig(fs afero.Fs, withConfig ...func(cfg config.Provider) error) (*viper.Viper, error) {
+ v, _, err := LoadConfig(ConfigSourceDescriptor{Fs: fs}, withConfig...)
+ return v, err
+}
+func newTestCfgBasic() (*viper.Viper, *hugofs.Fs) {
+ mm := afero.NewMemMapFs()
v := viper.New()
- fs := hugofs.NewMem(v)
+ v.Set("defaultContentLanguageInSubdir", true)
- v.SetFs(fs.Source)
+ fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(mm), v)
- loadDefaultSettingsFor(v)
+ return v, fs
- // Default is false, but true is easier to use as default in tests
- v.Set("defaultContentLanguageInSubdir", true)
+}
+
+func newTestCfg(withConfig ...func(cfg config.Provider) error) (*viper.Viper, *hugofs.Fs) {
+ mm := afero.NewMemMapFs()
+
+ v, err := loadTestConfig(mm, func(cfg config.Provider) error {
+ // Default is false, but true is easier to use as default in tests
+ cfg.Set("defaultContentLanguageInSubdir", true)
+
+ for _, w := range withConfig {
+ w(cfg)
+ }
+
+ return nil
+ })
+
+ if err != nil && err != ErrNoConfigFile {
+ panic(err)
+ }
+
+ fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(mm), v)
return v, fs
@@ -651,9 +760,10 @@ func newTestCfg() (*viper.Viper, *hugofs.Fs) {
func newTestSitesFromConfig(t testing.TB, afs afero.Fs, tomlConfig string, layoutPathContentPairs ...string) (testHelper, *HugoSites) {
if len(layoutPathContentPairs)%2 != 0 {
- Fatalf(t, "Layouts must be provided in pairs")
+ t.Fatalf("Layouts must be provided in pairs")
}
+ writeToFs(t, afs, filepath.Join("content", ".gitkeep"), "")
writeToFs(t, afs, "config.toml", tomlConfig)
cfg, err := LoadConfigDefault(afs)
@@ -673,14 +783,6 @@ func newTestSitesFromConfig(t testing.TB, afs afero.Fs, tomlConfig string, layou
return th, h
}
-func newTestSitesFromConfigWithDefaultTemplates(t testing.TB, tomlConfig string) (testHelper, *HugoSites) {
- return newTestSitesFromConfig(t, afero.NewMemMapFs(), tomlConfig,
- "layouts/_default/single.html", "Single|{{ .Title }}|{{ .Content }}",
- "layouts/_default/list.html", "List|{{ .Title }}|{{ .Content }}",
- "layouts/_default/terms.html", "Terms List|{{ .Title }}|{{ .Content }}",
- )
-}
-
func createWithTemplateFromNameValues(additionalTemplates ...string) func(templ tpl.TemplateHandler) error {
return func(templ tpl.TemplateHandler) error {
@@ -694,12 +796,16 @@ func createWithTemplateFromNameValues(additionalTemplates ...string) func(templ
}
}
+// TODO(bep) replace these with the builder
func buildSingleSite(t testing.TB, depsCfg deps.DepsCfg, buildCfg BuildCfg) *Site {
return buildSingleSiteExpected(t, false, false, depsCfg, buildCfg)
}
func buildSingleSiteExpected(t testing.TB, expectSiteInitEror, expectBuildError bool, depsCfg deps.DepsCfg, buildCfg BuildCfg) *Site {
- h, err := NewHugoSites(depsCfg)
+ t.Helper()
+ b := newTestSitesBuilderFromDepsCfg(t, depsCfg).WithNothingAdded()
+
+ err := b.CreateSitesE()
if expectSiteInitEror {
require.Error(t, err)
@@ -708,6 +814,8 @@ func buildSingleSiteExpected(t testing.TB, expectSiteInitEror, expectBuildError
require.NoError(t, err)
}
+ h := b.H
+
require.Len(t, h.Sites, 1)
if expectBuildError {
@@ -751,9 +859,13 @@ func content(c resource.ContentProvider) string {
func dumpPages(pages ...page.Page) {
fmt.Println("---------")
for i, p := range pages {
- fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s\n",
+ var meta interface{}
+ if p.File() != nil && p.File().FileInfo() != nil {
+ meta = p.File().FileInfo().Meta()
+ }
+ fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s Lang: %s Meta: %v\n",
i+1,
- p.Kind(), p.Title(), p.RelPermalink(), p.Path(), p.SectionsPath())
+ p.Kind(), p.Title(), p.RelPermalink(), p.Path(), p.SectionsPath(), p.Lang(), meta)
}
}
diff --git a/hugolib/testsite/.gitignore b/hugolib/testsite/.gitignore
new file mode 100644
index 00000000000..ab8b69cbc18
--- /dev/null
+++ b/hugolib/testsite/.gitignore
@@ -0,0 +1 @@
+config.toml
\ No newline at end of file
diff --git a/langs/config.go b/langs/config.go
new file mode 100644
index 00000000000..927f3558fa4
--- /dev/null
+++ b/langs/config.go
@@ -0,0 +1,217 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package langs
+
+import (
+ "fmt"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/gohugoio/hugo/common/maps"
+
+ "github.com/spf13/cast"
+
+ "github.com/pkg/errors"
+
+ "github.com/gohugoio/hugo/config"
+)
+
+type LanguagesConfig struct {
+ Languages Languages
+ Multihost bool
+ DefaultContentLanguageInSubdir bool
+}
+
+func LoadLanguageSettings(cfg config.Provider, oldLangs Languages) (c LanguagesConfig, err error) {
+
+ defaultLang := cfg.GetString("defaultContentLanguage")
+ if defaultLang == "" {
+ defaultLang = "en"
+ cfg.Set("defaultContentLanguage", defaultLang)
+ }
+
+ var languages map[string]interface{}
+
+ languagesFromConfig := cfg.GetStringMap("languages")
+ disableLanguages := cfg.GetStringSlice("disableLanguages")
+
+ if len(disableLanguages) == 0 {
+ languages = languagesFromConfig
+ } else {
+ languages = make(map[string]interface{})
+ for k, v := range languagesFromConfig {
+ for _, disabled := range disableLanguages {
+ if disabled == defaultLang {
+ return c, fmt.Errorf("cannot disable default language %q", defaultLang)
+ }
+
+ if strings.EqualFold(k, disabled) {
+ v.(map[string]interface{})["disabled"] = true
+ break
+ }
+ }
+ languages[k] = v
+ }
+ }
+
+ var languages2 Languages
+
+ if len(languages) == 0 {
+ languages2 = append(languages2, NewDefaultLanguage(cfg))
+ } else {
+ languages2, err = toSortedLanguages(cfg, languages)
+ if err != nil {
+ return c, errors.Wrap(err, "Failed to parse multilingual config")
+ }
+ }
+
+ if oldLangs != nil {
+ // When in multihost mode, the languages are mapped to a server, so
+ // some structural language changes will need a restart of the dev server.
+ // The validation below isn't complete, but should cover the most
+ // important cases.
+ var invalid bool
+ if languages2.IsMultihost() != oldLangs.IsMultihost() {
+ invalid = true
+ } else {
+ if languages2.IsMultihost() && len(languages2) != len(oldLangs) {
+ invalid = true
+ }
+ }
+
+ if invalid {
+ return c, errors.New("language change needing a server restart detected")
+ }
+
+ if languages2.IsMultihost() {
+ // We need to transfer any server baseURL to the new language
+ for i, ol := range oldLangs {
+ nl := languages2[i]
+ nl.Set("baseURL", ol.GetString("baseURL"))
+ }
+ }
+ }
+
+ // The defaultContentLanguage is something the user has to decide, but it needs
+ // to match a language in the language definition list.
+ langExists := false
+ for _, lang := range languages2 {
+ if lang.Lang == defaultLang {
+ langExists = true
+ break
+ }
+ }
+
+ if !langExists {
+ return c, fmt.Errorf("site config value %q for defaultContentLanguage does not match any language definition", defaultLang)
+ }
+
+ c.Languages = languages2
+ c.Multihost = languages2.IsMultihost()
+ c.DefaultContentLanguageInSubdir = c.Multihost
+
+ sortedDefaultFirst := make(Languages, len(c.Languages))
+ for i, v := range c.Languages {
+ sortedDefaultFirst[i] = v
+ }
+ sort.Slice(sortedDefaultFirst, func(i, j int) bool {
+ li, lj := sortedDefaultFirst[i], sortedDefaultFirst[j]
+ if li.Lang == defaultLang {
+ return true
+ }
+
+ if lj.Lang == defaultLang {
+ return false
+ }
+
+ return i < j
+ })
+
+ cfg.Set("languagesSorted", c.Languages)
+ cfg.Set("languagesSortedDefaultFirst", sortedDefaultFirst)
+ cfg.Set("multilingual", len(languages2) > 1)
+
+ multihost := c.Multihost
+
+ if multihost {
+ cfg.Set("defaultContentLanguageInSubdir", true)
+ cfg.Set("multihost", true)
+ }
+
+ if multihost {
+ // The baseURL may be provided at the language level. If that is true,
+ // then every language must have a baseURL. In this case we always render
+ // to a language sub folder, which is then stripped from all the Permalink URLs etc.
+ for _, l := range languages2 {
+ burl := l.GetLocal("baseURL")
+ if burl == nil {
+ return c, errors.New("baseURL must be set on all or none of the languages")
+ }
+ }
+
+ }
+
+ return c, nil
+}
+
+func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (Languages, error) {
+ languages := make(Languages, len(l))
+ i := 0
+
+ for lang, langConf := range l {
+ langsMap, err := cast.ToStringMapE(langConf)
+
+ if err != nil {
+ return nil, fmt.Errorf("Language config is not a map: %T", langConf)
+ }
+
+ language := NewLanguage(lang, cfg)
+
+ for loki, v := range langsMap {
+ switch loki {
+ case "title":
+ language.Title = cast.ToString(v)
+ case "languagename":
+ language.LanguageName = cast.ToString(v)
+ case "weight":
+ language.Weight = cast.ToInt(v)
+ case "contentdir":
+ language.ContentDir = filepath.Clean(cast.ToString(v))
+ case "disabled":
+ language.Disabled = cast.ToBool(v)
+ case "params":
+ m := cast.ToStringMap(v)
+ // Needed for case insensitive fetching of params values
+ maps.ToLower(m)
+ for k, vv := range m {
+ language.SetParam(k, vv)
+ }
+ }
+
+ // Put all into the Params map
+ language.SetParam(loki, v)
+
+ // Also set it in the configuration map (for baseURL etc.)
+ language.Set(loki, v)
+ }
+
+ languages[i] = language
+ i++
+ }
+
+ sort.Sort(languages)
+
+ return languages, nil
+}
diff --git a/langs/i18n/i18n_test.go b/langs/i18n/i18n_test.go
index b67cabc5524..e08210848fe 100644
--- a/langs/i18n/i18n_test.go
+++ b/langs/i18n/i18n_test.go
@@ -17,11 +17,13 @@ import (
"path/filepath"
"testing"
+ "github.com/gohugoio/hugo/modules"
+
"github.com/gohugoio/hugo/tpl/tplimpl"
"github.com/gohugoio/hugo/common/loggers"
- "github.com/gohugoio/hugo/htesting"
"github.com/gohugoio/hugo/langs"
+ "github.com/gohugoio/hugo/resources/page"
"github.com/spf13/afero"
"github.com/spf13/viper"
@@ -199,7 +201,7 @@ func newDepsConfig(tp *TranslationProvider, cfg config.Provider, fs *hugofs.Fs)
l.Set("i18nDir", "i18n")
return deps.DepsCfg{
Language: l,
- Site: htesting.NewTestHugoSite(),
+ Site: page.NewDummyHugoSite(cfg),
Cfg: cfg,
Fs: fs,
Logger: logger,
@@ -219,6 +221,13 @@ func getConfig() *viper.Viper {
v.Set("assetDir", "assets")
v.Set("resourceDir", "resources")
v.Set("publishDir", "public")
+ langs.LoadLanguageSettings(v, nil)
+ mod, err := modules.CreateProjectModule(v)
+ if err != nil {
+ panic(err)
+ }
+ v.Set("allModules", modules.Modules{mod})
+
return v
}
diff --git a/langs/i18n/translationProvider.go b/langs/i18n/translationProvider.go
index 74e144007ef..c7b4839eef1 100644
--- a/langs/i18n/translationProvider.go
+++ b/langs/i18n/translationProvider.go
@@ -40,8 +40,7 @@ func NewTranslationProvider() *TranslationProvider {
// Update updates the i18n func in the provided Deps.
func (tp *TranslationProvider) Update(d *deps.Deps) error {
- sp := source.NewSourceSpec(d.PathSpec, d.BaseFs.SourceFilesystems.I18n.Fs)
- src := sp.NewFilesystem("")
+ spec := source.NewSourceSpec(d.PathSpec, nil)
i18nBundle := bundle.New()
@@ -51,25 +50,33 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error {
}
var newLangs []string
- for _, r := range src.Files() {
- currentSpec := language.GetPluralSpec(r.BaseFileName())
- if currentSpec == nil {
- // This may is a language code not supported by go-i18n, it may be
- // Klingon or ... not even a fake language. Make sure it works.
- newLangs = append(newLangs, r.BaseFileName())
+ for _, dir := range d.BaseFs.I18n.Dirs {
+ src := spec.NewFilesystemFromFileMetaInfo(dir)
+
+ files, err := src.Files()
+ if err != nil {
+ return err
}
- }
- if len(newLangs) > 0 {
- language.RegisterPluralSpec(newLangs, en)
- }
+ for _, r := range files {
+ currentSpec := language.GetPluralSpec(r.BaseFileName())
+ if currentSpec == nil {
+ // This may is a language code not supported by go-i18n, it may be
+ // Klingon or ... not even a fake language. Make sure it works.
+ newLangs = append(newLangs, r.BaseFileName())
+ }
+ }
- // The source files are ordered so the most important comes first. Since this is a
- // last key win situation, we have to reverse the iteration order.
- files := src.Files()
- for i := len(files) - 1; i >= 0; i-- {
- if err := addTranslationFile(i18nBundle, files[i]); err != nil {
- return err
+ if len(newLangs) > 0 {
+ language.RegisterPluralSpec(newLangs, en)
+ }
+
+ // The source files are ordered so the most important comes first. Since this is a
+ // last key win situation, we have to reverse the iteration order.
+ for i := len(files) - 1; i >= 0; i-- {
+ if err := addTranslationFile(i18nBundle, files[i]); err != nil {
+ return err
+ }
}
}
@@ -81,8 +88,8 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error {
}
-func addTranslationFile(bundle *bundle.Bundle, r source.ReadableFile) error {
- f, err := r.Open()
+func addTranslationFile(bundle *bundle.Bundle, r source.File) error {
+ f, err := r.FileInfo().Meta().Open()
if err != nil {
return _errors.Wrapf(err, "failed to open translations file %q:", r.LogicalName())
}
@@ -101,14 +108,15 @@ func (tp *TranslationProvider) Clone(d *deps.Deps) error {
return nil
}
-func errWithFileContext(inerr error, r source.ReadableFile) error {
- rfi, ok := r.FileInfo().(hugofs.RealFilenameInfo)
+func errWithFileContext(inerr error, r source.File) error {
+ fim, ok := r.FileInfo().(hugofs.FileMetaInfo)
if !ok {
return inerr
}
- realFilename := rfi.RealFilename()
- f, err := r.Open()
+ meta := fim.Meta()
+ realFilename := meta.Filename()
+ f, err := meta.Open()
if err != nil {
return inerr
}
diff --git a/langs/language.go b/langs/language.go
index 14e3263aeb9..f71b0255b38 100644
--- a/langs/language.go
+++ b/langs/language.go
@@ -78,12 +78,7 @@ func NewLanguage(lang string, cfg config.Provider) *Language {
}
maps.ToLower(params)
- defaultContentDir := cfg.GetString("contentDir")
- if defaultContentDir == "" {
- panic("contentDir not set")
- }
-
- l := &Language{Lang: lang, ContentDir: defaultContentDir, Cfg: cfg, params: params, settings: make(map[string]interface{})}
+ l := &Language{Lang: lang, ContentDir: cfg.GetString("contentDir"), Cfg: cfg, params: params, settings: make(map[string]interface{})}
return l
}
@@ -132,6 +127,24 @@ func (l *Language) Params() map[string]interface{} {
return l.params
}
+func (l Languages) AsSet() map[string]bool {
+ m := make(map[string]bool)
+ for _, lang := range l {
+ m[lang.Lang] = true
+ }
+
+ return m
+}
+
+func (l Languages) AsOrdinalSet() map[string]int {
+ m := make(map[string]int)
+ for i, lang := range l {
+ m[lang.Lang] = i
+ }
+
+ return m
+}
+
// IsMultihost returns whether there are more than one language and at least one of
// the languages has baseURL specificed on the language level.
func (l Languages) IsMultihost() bool {
diff --git a/magefile.go b/magefile.go
index 3b74a7e940e..d0b7c8d98a6 100644
--- a/magefile.go
+++ b/magefile.go
@@ -143,20 +143,31 @@ func Check() {
mg.Deps(TestRace)
}
+func testGoFlags() string {
+ if isCI() {
+ return ""
+ }
+
+ return "-test.short"
+}
+
// Run tests in 32-bit mode
// Note that we don't run with the extended tag. Currently not supported in 32 bit.
func Test386() error {
- return sh.RunWith(map[string]string{"GOARCH": "386"}, goexe, "test", "./...")
+ env := map[string]string{"GOARCH": "386", "GOFLAGS": testGoFlags()}
+ return sh.RunWith(env, goexe, "test", "./...")
}
// Run tests
func Test() error {
- return sh.Run(goexe, "test", "./...", "-tags", buildTags())
+ env := map[string]string{"GOFLAGS": testGoFlags()}
+ return sh.RunWith(env, goexe, "test", "./...", "-tags", buildTags())
}
// Run tests with race detector
func TestRace() error {
- return sh.Run(goexe, "test", "-race", "./...", "-tags", buildTags())
+ env := map[string]string{"GOFLAGS": testGoFlags()}
+ return sh.RunWith(env, goexe, "test", "-race", "./...", "-tags", buildTags())
}
// Run gofmt linter
@@ -296,6 +307,10 @@ func isGoLatest() bool {
return strings.Contains(runtime.Version(), "1.11")
}
+func isCI() bool {
+ return os.Getenv("CI") != ""
+}
+
func buildTags() string {
// To build the extended Hugo SCSS/SASS enabled version, build with
// HUGO_BUILD_TAGS=extended mage install etc.
diff --git a/modules/client.go b/modules/client.go
new file mode 100644
index 00000000000..76e207fb788
--- /dev/null
+++ b/modules/client.go
@@ -0,0 +1,593 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/rogpeppe/go-internal/module"
+
+ "github.com/gohugoio/hugo/common/hugio"
+
+ "github.com/pkg/errors"
+ "github.com/spf13/afero"
+)
+
+var (
+ fileSeparator = string(os.PathSeparator)
+)
+
+const hugoModProxyEnvKey = "HUGO_MODPROXY"
+
+const (
+ goBinaryStatusOK goBinaryStatus = iota
+ goBinaryStatusNotFound
+ goBinaryStatusTooOld
+)
+
+// The "vendor" dir is reserved for Go Modules.
+const vendord = "_vendor"
+
+// These are the folders we consider to be part of a module when we vendor
+// it.
+// TODO(bep) mod configurable...? regexp?
+var dirnames = map[string]bool{
+ "archetypes": true,
+ "assets": true,
+ "data": true,
+ "i18n": true,
+ "layouts": true,
+ "resources": true,
+ "static": true,
+}
+
+const (
+ goModFilename = "go.mod"
+ goSumFilename = "go.sum"
+)
+
+type ClientConfig struct {
+ Fs afero.Fs
+ IgnoreVendor bool
+ WorkingDir string
+ ThemesDir string // Absolute directory path
+ ModProxy string
+ ModuleConfig Config
+}
+
+// TODO(bep) mod document modProxy config + HUGO_MODPROXY
+func (c ClientConfig) getGoProxy() string {
+ if c.ModProxy != "" {
+ return c.ModProxy
+ }
+ // Default to direct, which means "git clone" and similar. We
+ // will investigate proxy settings in more depth later.
+ // See https://github.com/golang/go/issues/26334
+ return "direct"
+}
+
+// NewClient creates a new Client that can be used to manage the Hugo Components
+// in a given workingDir.
+// The Client will resolve the dependencies recursively, but needs the top
+// level imports to start out.
+func NewClient(cfg ClientConfig) *Client {
+ fs := cfg.Fs
+
+ n := filepath.Join(cfg.WorkingDir, goModFilename)
+ goModEnabled, _ := afero.Exists(fs, n)
+ var goModFilename string
+ if goModEnabled {
+ goModFilename = n
+ }
+
+ /*
+
+ TODO(bep) mod consider options for these.
+
+ GOPRIVATE=*.corp.example.com
+ GOPROXY=proxy.example.com
+ GONOPROXY=none
+
+ */
+
+ env := os.Environ()
+ setEnvVars(&env, "PWD", cfg.WorkingDir, "GOPROXY", getGoProxy(), "GOPRIVATE", "*.*")
+
+ return &Client{
+ fs: fs,
+ ignoreVendor: cfg.IgnoreVendor,
+ workingDir: cfg.WorkingDir,
+ themesDir: cfg.ThemesDir,
+ moduleConfig: cfg.ModuleConfig,
+ environ: env,
+ GoModulesFilename: goModFilename}
+}
+
+// Client contains most of the API provided by this package.
+type Client struct {
+ fs afero.Fs
+
+ // Ignore any _vendor directory.
+ ignoreVendor bool
+
+ // Absolute path to the project dir.
+ workingDir string
+
+ // Absolute path to the project's themes dir.
+ themesDir string
+
+ // The top level module config
+ moduleConfig Config
+
+ // Environment variables used in "go get" etc.
+ environ []string
+
+ // Set when Go modules are initialized in the current repo, that is:
+ // a go.mod file exists.
+ GoModulesFilename string
+
+ // Set if we get a exec.ErrNotFound when running Go, which is most likely
+ // due to being run on a system without Go installed. We record it here
+ // so we can give an instructional error at the end if module/theme
+ // resolution fails.
+ goBinaryStatus goBinaryStatus
+}
+
+// TODO(bep) mod probably filter this against imports? Also check replace.
+// TODO(bep) merge with _vendor + /theme
+func (m *Client) Graph(w io.Writer) error {
+ mc, err := m.Collect()
+ if err != nil {
+ return err
+ }
+ for _, module := range mc.Modules {
+ if module.Owner() == nil {
+ continue
+ }
+ dep := pathVersion(module.Owner()) + " " + pathVersion(module)
+ if replace := module.Replace(); replace != nil {
+ dep += " => " + replace.Dir()
+ }
+ fmt.Fprintln(w, dep)
+
+ }
+
+ return nil
+}
+
+// Tidy can be used to remove unused dependencies from go.mod and go.sum.
+func (m *Client) Tidy() error {
+ tc, err := m.Collect()
+ if err != nil {
+ return err
+ }
+
+ isGoMod := make(map[string]bool)
+ for _, m := range tc.Modules {
+ if m.Owner() == nil {
+ continue
+ }
+ if m.IsGoMod() {
+ // Matching the format in go.mod
+ isGoMod[m.Path()+" "+m.Version()] = true
+ }
+ }
+
+ if err := m.rewriteGoMod(goModFilename, isGoMod); err != nil {
+ return err
+ }
+
+ // Now go.mod contains only in-use modules. The go.sum file will
+ // contain the entire dependency graph, so we need to check against that.
+ // TODO(bep) check if needed
+ /*graph, err := m.graphStr()
+ if err != nil {
+ return err
+ }
+
+ isGoMod = make(map[string]bool)
+ graphItems := strings.Split(graph, "\n")
+ for _, item := range graphItems {
+ item = strings.TrimSpace(item)
+ if item == "" {
+ continue
+ }
+ modver := strings.Replace(strings.Fields(item)[1], "@", " ", 1)
+ isGoMod[modver] = true
+ }*/
+
+ if err := m.rewriteGoMod(goSumFilename, isGoMod); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (m *Client) Get(args ...string) error {
+ if err := m.runGo(context.Background(), os.Stdout, append([]string{"get"}, args...)...); err != nil {
+ errors.Wrapf(err, "failed to get %q", args)
+ }
+ return nil
+}
+
+func (m *Client) Init(path string) error {
+
+ err := m.runGo(context.Background(), os.Stdout, "mod", "init", path)
+ if err != nil {
+ return errors.Wrap(err, "failed to init modules")
+ }
+
+ m.GoModulesFilename = filepath.Join(m.workingDir, goModFilename)
+
+ return nil
+}
+
+func (m *Client) IsProbablyModule(path string) bool {
+ return module.CheckPath(path) == nil
+}
+
+// Like Go, Hugo supports writing the dependencies to a /_vendor folder.
+// Unlike Go, we support it for any level.
+// We, by defaults, use the /_vendor folder first, if found. To disable,
+// run with
+// hugo --ignoreVendor
+//
+// Given a module tree, Hugo will pick the first module for a given path,
+// meaning that if the top-level module is vendored, that will be the full
+// set of dependencies.
+func (c *Client) Vendor() error {
+ vendorDir := filepath.Join(c.workingDir, vendord)
+ if err := c.rmVendorDir(vendorDir); err != nil {
+ return err
+ }
+
+ // Write the modules list to modules.txt.
+ //
+ // On the form:
+ //
+ // # github.com/alecthomas/chroma v0.6.3
+ //
+ // This is how "go mod vendor" does it. Go also lists
+ // the packages below it, but that is currently not applicable to us.
+ //
+ var modulesContent bytes.Buffer
+
+ tc, err := c.Collect()
+ if err != nil {
+ return err
+ }
+
+ for _, t := range tc.Modules {
+ if t.Owner() == nil {
+ // This is the project.
+ continue
+ }
+ // We respect the --ignoreVendor flag even for the vendor command.
+ if !t.IsGoMod() && !t.Vendor() {
+ // We currently do not vendor components living in the
+ // theme directory, see https://github.com/gohugoio/hugo/issues/5993
+ continue
+ }
+
+ fmt.Fprintln(&modulesContent, "# "+t.Path()+" "+t.Version())
+
+ dir := t.Dir()
+
+ shouldCopy := func(filename string) bool {
+ //base := filepath.Base(strings.TrimPrefix(filename, dir))
+ // Only vendor the root files + the predefined set of folders.
+ // TODO(bep) mod fix me, only root
+ return true // base != "_vendor" //dirnames[base]
+ }
+
+ if err := hugio.CopyDir(c.fs, dir, filepath.Join(vendorDir, t.Path()), shouldCopy); err != nil {
+ return errors.Wrap(err, "failed to copy module to vendor dir")
+ }
+ }
+
+ if modulesContent.Len() > 0 {
+ if err := afero.WriteFile(c.fs, filepath.Join(vendorDir, vendorModulesFilename), modulesContent.Bytes(), 0666); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (m *Client) listGoMods() (goModules, error) {
+ if m.GoModulesFilename == "" {
+ return nil, nil
+ }
+ ///
+ // TODO(bep) mod check permissions
+ // TODO(bep) mod clear cache
+ // TODO(bep) mount at all of partials/ partials/v1 partials/v2 or something.
+ // TODO(bep) rm: public/images/logos/made-with-bulma.png: Permission denied
+ // TODO(bep) watch pkg cache?
+ // 0555 directories
+ // TODO(bep) mod hugo mod init
+ // GO111MODULE=on
+ //
+
+ // TODO(bep) mod --no-vendor flag (also on hugo)
+ // TODO(bep) mod hugo mod vendor: --no-local
+ // GOCACHE
+
+ out := ioutil.Discard
+ err := m.runGo(context.Background(), out, "mod", "download")
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to download modules")
+ }
+
+ b := &bytes.Buffer{}
+ err = m.runGo(context.Background(), b, "list", "-m", "-json", "all")
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to list modules")
+ }
+
+ var modules goModules
+
+ dec := json.NewDecoder(b)
+ for {
+ m := &goModule{}
+ if err := dec.Decode(m); err != nil {
+ if err == io.EOF {
+ break
+ }
+ return nil, errors.Wrap(err, "failed to decode modules list")
+ }
+
+ modules = append(modules, m)
+ }
+
+ return modules, err
+
+}
+
+func (m *Client) rewriteGoMod(name string, isGoMod map[string]bool) error {
+ data, err := m.rewriteGoModRewrite(name, isGoMod)
+ if err != nil {
+ return err
+ }
+ if data != nil {
+ // Rewrite the file.
+ if err := afero.WriteFile(m.fs, filepath.Join(m.workingDir, name), data, 0666); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (m *Client) rewriteGoModRewrite(name string, isGoMod map[string]bool) ([]byte, error) {
+ if name == goModFilename && m.GoModulesFilename == "" {
+ // Already checked.
+ return nil, nil
+ }
+
+ isModLine := func(s string) bool {
+ return true
+ }
+
+ if name == goModFilename {
+ isModLine = func(s string) bool {
+ return strings.HasPrefix(s, "mod require") || strings.HasPrefix(s, "\t")
+ }
+ }
+
+ b := &bytes.Buffer{}
+ f, err := m.fs.Open(filepath.Join(m.workingDir, name))
+ if err != nil {
+ if os.IsNotExist(err) {
+ // It's been deleted.
+ return nil, nil
+ }
+ return nil, err
+ }
+ defer f.Close()
+
+ scanner := bufio.NewScanner(f)
+ var dirty bool
+
+ for scanner.Scan() {
+ line := scanner.Text()
+ var doWrite bool
+
+ if isModLine(line) {
+ modname := strings.TrimSpace(line)
+ if modname == "" {
+ doWrite = true
+ } else {
+ // TODO(bep) mod: mod require
+ parts := strings.Fields(modname)
+ if len(parts) >= 2 {
+ // [module path] [version]/go.mod
+ modname, modver := parts[0], parts[1]
+ modver = strings.TrimSuffix(modver, "/"+goModFilename)
+ doWrite = isGoMod[modname+" "+modver]
+ }
+ }
+ } else {
+ doWrite = true
+ }
+
+ if doWrite {
+ fmt.Fprintln(b, line)
+ } else {
+ dirty = true
+ }
+ }
+
+ if !dirty {
+ // Nothing changed
+ return nil, nil
+ }
+
+ return b.Bytes(), nil
+
+}
+
+func (c *Client) rmVendorDir(vendorDir string) error {
+ modulestxt := filepath.Join(vendorDir, vendorModulesFilename)
+
+ if _, err := c.fs.Stat(vendorDir); err != nil {
+ return nil
+ }
+
+ _, err := c.fs.Stat(modulestxt)
+ if err != nil {
+ // If we have a _vendor dir without modules.txt it sounds like
+ // a _vendor dir created by others.
+ return errors.New("found _vendor dir without modules.txt, skip delete")
+ }
+
+ return c.fs.RemoveAll(vendorDir)
+}
+
+func (m *Client) runGo(
+ ctx context.Context,
+ stdout io.Writer,
+ args ...string) error {
+
+ if m.goBinaryStatus != 0 {
+ return nil
+ }
+
+ stderr := new(bytes.Buffer)
+ cmd := exec.CommandContext(ctx, "go", args...)
+
+ cmd.Env = m.environ
+ cmd.Dir = m.workingDir
+ cmd.Stdout = stdout
+ cmd.Stderr = io.MultiWriter(stderr, os.Stderr)
+
+ if err := cmd.Run(); err != nil {
+ if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound {
+ m.goBinaryStatus = goBinaryStatusNotFound
+ return nil
+ }
+
+ _, ok := err.(*exec.ExitError)
+ if !ok {
+ return errors.Errorf("failed to execute 'go %v': %s %T", args, err, err)
+ }
+
+ // Too old Go version
+ if strings.Contains(stderr.String(), "flag provided but not defined") {
+ m.goBinaryStatus = goBinaryStatusTooOld
+ return nil
+ }
+
+ return errors.Errorf("go command failed: %s", stderr)
+
+ }
+
+ return nil
+}
+
+type ModuleError struct {
+ Err string // the error itself
+}
+
+type goBinaryStatus int
+
+type goModule struct {
+ Path string // module path
+ Version string // module version
+ Versions []string // available module versions (with -versions)
+ Replace *goModule // replaced by this module
+ Time *time.Time // time version was created
+ Update *goModule // available update, if any (with -u)
+ Main bool // is this the main module?
+ Indirect bool // is this module only an indirect dependency of main module?
+ Dir string // directory holding files for this module, if any
+ GoMod string // path to go.mod file for this module, if any
+ Error *ModuleError // error loading module
+}
+
+type goModules []*goModule
+
+func (modules goModules) GetByPath(p string) *goModule {
+ if modules == nil {
+ return nil
+ }
+
+ for _, m := range modules {
+ if strings.EqualFold(p, m.Path) {
+ return m
+ }
+ }
+
+ return nil
+}
+
+func (modules goModules) GetMain() *goModule {
+ for _, m := range modules {
+ if m.Main {
+ return m
+ }
+ }
+
+ return nil
+}
+
+func setEnvVar(vars *[]string, key, value string) {
+ for i := range *vars {
+ if strings.HasPrefix((*vars)[i], key+"=") {
+ (*vars)[i] = key + "=" + value
+ return
+ }
+ }
+ // New var.
+ *vars = append(*vars, key+"="+value)
+}
+
+func setEnvVars(oldVars *[]string, keyValues ...string) {
+ for i := 0; i < len(keyValues); i += 2 {
+ setEnvVar(oldVars, keyValues[i], keyValues[i+1])
+ }
+}
+
+func getGoProxy() string {
+ if hp := os.Getenv(hugoModProxyEnvKey); hp != "" {
+ return hp
+ }
+
+ // Default to direct, which means "git clone" and similar. We
+ // will investigate proxy settings in more depth later.
+ // See https://github.com/golang/go/issues/26334
+ return "direct"
+}
+
+func pathVersion(m Module) string {
+ versionStr := m.Version()
+ if m.Vendor() {
+ versionStr += "+vendor"
+ }
+ if versionStr == "" {
+ return m.Path()
+ }
+ return fmt.Sprintf("%s@%s", m.Path(), versionStr)
+}
diff --git a/modules/client_test.go b/modules/client_test.go
new file mode 100644
index 00000000000..4b705892e34
--- /dev/null
+++ b/modules/client_test.go
@@ -0,0 +1,112 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+ "bytes"
+ "testing"
+
+ "github.com/gohugoio/hugo/common/hugo"
+
+ "github.com/gohugoio/hugo/htesting"
+
+ "github.com/gohugoio/hugo/hugofs"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestClient(t *testing.T) {
+ if hugo.GoMinorVersion() < 12 {
+ // https://github.com/golang/go/issues/26794
+ // There were some concurrent issues with Go modules in < Go 12.
+ t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib")
+ }
+
+ t.Parallel()
+
+ modName := "hugo-modules-basic-test"
+ modPath := "github.com/gohugoio/tests/" + modName
+ modConfig := Config{
+ Imports: []Import{Import{Path: "github.com/gohugoio/hugoTestModules1_darwin/modh2_2"}},
+ }
+
+ assert := require.New(t)
+
+ workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, modName)
+ assert.NoError(err)
+ defer clean()
+
+ client := NewClient(ClientConfig{
+ Fs: hugofs.Os,
+ WorkingDir: workingDir,
+ ModuleConfig: modConfig,
+ })
+
+ // Test Init
+ assert.NoError(client.Init(modPath))
+
+ // Test Collect
+ mc, err := client.Collect()
+ assert.NoError(err)
+ assert.Equal(4, len(mc.Modules))
+ for _, m := range mc.Modules {
+ assert.NotNil(m)
+ }
+
+ // Test Graph
+ var graphb bytes.Buffer
+ assert.NoError(client.Graph(&graphb))
+
+ expect := `github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0
+github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2_1v@v1.3.0
+github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0
+`
+
+ assert.Equal(expect, graphb.String())
+
+ // Test Vendor
+ assert.NoError(client.Vendor())
+ graphb.Reset()
+ assert.NoError(client.Graph(&graphb))
+ expectVendored := `github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0+vendor
+github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2_1v@v1.3.0+vendor
+github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0+vendor
+`
+ assert.Equal(expectVendored, graphb.String())
+
+ // Test the ignoreVendor setting
+ clientIgnoreVendor := NewClient(ClientConfig{
+ Fs: hugofs.Os,
+ WorkingDir: workingDir,
+ ModuleConfig: modConfig,
+ IgnoreVendor: true,
+ })
+
+ graphb.Reset()
+ assert.NoError(clientIgnoreVendor.Graph(&graphb))
+ assert.Equal(expect, graphb.String())
+
+ // Test Tidy
+ // TODO(bep) improve
+ assert.NoError(client.Tidy())
+
+}
+
+func TestSetEnvVars(t *testing.T) {
+ t.Parallel()
+ assert := require.New(t)
+ vars := []string{"FOO=bar", "HUGO=cool", "BAR=foo"}
+ setEnvVars(&vars, "HUGO", "rocking!", "NEW", "bar")
+ assert.Equal([]string{"FOO=bar", "HUGO=rocking!", "BAR=foo", "NEW=bar"}, vars)
+}
diff --git a/modules/collect.go b/modules/collect.go
new file mode 100644
index 00000000000..5a4dd9010e3
--- /dev/null
+++ b/modules/collect.go
@@ -0,0 +1,499 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+ "bufio"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/gohugoio/hugo/hugofs/files"
+
+ "github.com/rogpeppe/go-internal/module"
+
+ "github.com/pkg/errors"
+
+ "github.com/gohugoio/hugo/config"
+ "github.com/spf13/afero"
+)
+
+/*
+
+TODO(bep) mod consider this, also re. failing tip test.
+
+The go command knows the public key of sum.golang.org; use of any other
+database requires giving the public key explicitly. The URL defaults to
+"https://" followed by the database name.
+
+GOSUMDB defaults to "sum.golang.org", the Go checksum database run by Google.
+See https://sum.golang.org/privacy for the service's privacy policy.
+
+If GOSUMDB is set to "off", or if "go get" is invoked with the -insecure flag,
+the checksum database is not consulted, and all unrecognized modules are
+accepted, at the cost of giving up the security guarantee of verified repeatable
+downloads for all modules. A better way to bypass the checksum database
+for specific modules is to use the GOPRIVATE or GONOSUMDB environment
+variables. See 'go help module-private' for details.
+
+
+*/
+
+const vendorModulesFilename = "modules.txt"
+
+func (h *Client) hasImports() bool {
+ return len(h.moduleConfig.Imports) > 0
+}
+
+func (h *Client) Collect() (ModulesConfig, error) {
+ c := &collector{
+ Client: h,
+ }
+
+ if err := c.collect(); err != nil {
+ return ModulesConfig{}, err
+ }
+
+ return ModulesConfig{
+ Modules: c.modules,
+ GoModulesFilename: c.GoModulesFilename,
+ }, nil
+
+}
+
+type ModulesConfig struct {
+ Modules Modules
+
+ // Set if this is a Go modules enabled project.
+ GoModulesFilename string
+}
+
+type collected struct {
+ // Pick the first and prevent circular loops.
+ seen map[string]bool
+
+ // Maps module path to a _vendor dir. These values are fetched from
+ // _vendor/modules.txt, and the first (top-most) will win.
+ vendored map[string]vendoredModule
+
+ // Set if a Go modules enabled project.
+ gomods goModules
+
+ // Ordered list of collected modules, including Go Modules and theme
+ // components stored below /themes.
+ modules Modules
+}
+
+// Collects and creates a module tree.
+type collector struct {
+ *Client
+
+ *collected
+}
+
+type vendoredModule struct {
+ Owner Module
+ Dir string
+ Version string
+}
+
+func (c *collector) initModules() error {
+ c.collected = &collected{
+ seen: make(map[string]bool),
+ vendored: make(map[string]vendoredModule),
+ }
+
+ // We may fail later if we don't find the mods.
+ return c.loadModules()
+}
+
+// TODO(bep) mod:
+// - no-vendor
+func (c *collector) isSeen(path string) bool {
+ key := pathKey(path)
+ if c.seen[key] {
+ return true
+ }
+ c.seen[key] = true
+ return false
+}
+
+func (c *collector) getVendoredDir(path string) (vendoredModule, bool) {
+ v, found := c.vendored[path]
+ return v, found
+}
+
+// TODO(bep) mod
+const zeroVersion = ""
+
+func (c *collector) add(owner *moduleAdapter, moduleImport Import) (*moduleAdapter, error) {
+ var (
+ mod *goModule
+ moduleDir string
+ version string
+ vendored bool
+ )
+
+ modulePath := moduleImport.Path
+ var realOwner Module = owner
+
+ if !c.ignoreVendor {
+ if err := c.collectModulesTXT(owner); err != nil {
+ return nil, err
+ }
+
+ // Try _vendor first.
+ var vm vendoredModule
+ vm, vendored = c.getVendoredDir(modulePath)
+ if vendored {
+ moduleDir = vm.Dir
+ realOwner = vm.Owner
+ version = vm.Version
+
+ }
+ }
+
+ if moduleDir == "" {
+ mod = c.gomods.GetByPath(modulePath)
+ if mod != nil {
+ moduleDir = mod.Dir
+ }
+
+ if moduleDir == "" {
+
+ if c.GoModulesFilename != "" && c.IsProbablyModule(modulePath) {
+ // Try to "go get" it and reload the module configuration.
+ if err := c.Get(modulePath); err != nil {
+ return nil, err
+ }
+ if err := c.loadModules(); err != nil {
+ return nil, err
+ }
+
+ mod = c.gomods.GetByPath(modulePath)
+ if mod != nil {
+ moduleDir = mod.Dir
+ }
+ }
+
+ // Fall back to /themes/
+ if moduleDir == "" {
+ moduleDir = filepath.Join(c.themesDir, modulePath)
+
+ if found, _ := afero.Exists(c.fs, moduleDir); !found {
+ return nil, c.wrapModuleNotFound(errors.Errorf("module %q not found; either add it as a Hugo Module or store it in %q.", modulePath, c.themesDir))
+ }
+ }
+ }
+ }
+
+ if found, _ := afero.Exists(c.fs, moduleDir); !found {
+ return nil, c.wrapModuleNotFound(errors.Errorf("%q not found", moduleDir))
+ }
+
+ if !strings.HasSuffix(moduleDir, fileSeparator) {
+ moduleDir += fileSeparator
+ }
+
+ ma := &moduleAdapter{
+ dir: moduleDir,
+ vendor: vendored,
+ gomod: mod,
+ version: version,
+ // This may be the owner of the _vendor dir
+ owner: realOwner,
+ }
+ if mod == nil {
+ ma.path = modulePath
+ }
+
+ if err := ma.validateAndApplyDefaults(c.fs); err != nil {
+ return nil, err
+ }
+
+ if err := c.applyThemeConfig(ma); err != nil {
+ return nil, err
+ }
+
+ if err := c.applyMounts(moduleImport, ma); err != nil {
+ return nil, err
+ }
+
+ c.modules = append(c.modules, ma)
+ return ma, nil
+
+}
+
+func (c *collector) addAndRecurse(owner *moduleAdapter) error {
+ moduleConfig := owner.Config()
+ if owner.projectMod {
+ if err := c.applyMounts(Import{}, owner); err != nil {
+ return err
+ }
+ }
+
+ for _, moduleImport := range moduleConfig.Imports {
+ if !c.isSeen(moduleImport.Path) {
+ tc, err := c.add(owner, moduleImport)
+ if err != nil {
+ return err
+ }
+ if err := c.addAndRecurse(tc); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func (c *collector) applyMounts(moduleImport Import, mod *moduleAdapter) error {
+ mounts := moduleImport.Mounts
+
+ if !mod.projectMod && len(mounts) == 0 {
+ modConfig := mod.Config()
+ mounts = modConfig.Mounts
+ if len(mounts) == 0 {
+ // Create default mount points for every component folder that
+ // exists in the module.
+ for _, componentFolder := range files.ComponentFolders {
+ sourceDir := filepath.Join(mod.Dir(), componentFolder)
+ _, err := c.fs.Stat(sourceDir)
+ if err == nil {
+ mounts = append(mounts, Mount{
+ Source: componentFolder,
+ Target: componentFolder,
+ })
+ }
+ }
+ }
+ }
+
+ var err error
+ mounts, err = c.normalizeMounts(mod, mounts)
+ if err != nil {
+ return err
+ }
+
+ mod.mounts = mounts
+ return nil
+}
+
+func (c *collector) normalizeMounts(owner Module, mounts []Mount) ([]Mount, error) {
+ var out []Mount
+ dir := owner.Dir()
+
+ for _, mnt := range mounts {
+ errMsg := fmt.Sprintf("invalid module config for %q", owner.Path())
+
+ if mnt.Source == "" || mnt.Target == "" {
+ return nil, errors.New(errMsg + ": both source and target must be set")
+ }
+
+ mnt.Source = filepath.Clean(mnt.Source)
+ mnt.Target = filepath.Clean(mnt.Target)
+
+ // Verify that Source exists
+ sourceDir := filepath.Join(dir, mnt.Source)
+ _, err := c.fs.Stat(sourceDir)
+ if err != nil {
+ //fmt.Println(">>> ::::: NOT FOUND", sourceDir)
+
+ continue
+ //return nil, errors.Errorf("%s: module mount source not found: %q", errMsg, sourceDir)
+ }
+
+ // Verify that target points to one of the predefined component dirs
+ targetBase := mnt.Target
+ idxPathSep := strings.Index(mnt.Target, string(os.PathSeparator))
+ if idxPathSep != -1 {
+ targetBase = mnt.Target[0:idxPathSep]
+ }
+ if !files.IsComponentFolder(targetBase) {
+ return nil, errors.Errorf("%s: mount target must be one of: %v", errMsg, files.ComponentFolders)
+ }
+
+ out = append(out, mnt)
+ }
+
+ return out, nil
+}
+
+func (c *collector) applyThemeConfig(tc *moduleAdapter) error {
+
+ var (
+ configFilename string
+ cfg config.Provider
+ exists bool
+ )
+
+ // Viper supports more, but this is the sub-set supported by Hugo.
+ for _, configFormats := range config.ValidConfigFileExtensions {
+ configFilename = filepath.Join(tc.Dir(), "config."+configFormats)
+ exists, _ = afero.Exists(c.fs, configFilename)
+ if exists {
+ break
+ }
+ }
+
+ if !exists {
+ // No theme config set.
+ return nil
+ }
+
+ if configFilename != "" {
+ var err error
+ cfg, err = config.FromFile(c.fs, configFilename)
+ if err != nil {
+ return err
+ }
+ }
+
+ tc.configFilename = configFilename
+ tc.cfg = cfg
+
+ config, err := DecodeConfig(cfg)
+ if err != nil {
+ return err
+ }
+ tc.config = config
+
+ return nil
+
+}
+
+// CreateProjectModule creates modules from the given config.
+// This is used in tests only.
+func CreateProjectModule(cfg config.Provider) (Module, error) {
+ workingDir := cfg.GetString("workingDir")
+ var modConfig Config
+
+ mod := createProjectModule(nil, workingDir, modConfig)
+ if err := ApplyProjectConfigDefaults(cfg, mod); err != nil {
+ return nil, err
+ }
+
+ return mod, nil
+}
+
+func createProjectModule(gomod *goModule, workingDir string, conf Config) *moduleAdapter {
+ // Create a pseudo module for the main project.
+ var path string
+ if gomod == nil {
+ path = "project"
+ }
+
+ return &moduleAdapter{
+ path: path,
+ dir: workingDir,
+ gomod: gomod,
+ projectMod: true,
+ config: conf,
+ }
+
+}
+
+func (c *collector) collect() error {
+ if err := c.initModules(); err != nil {
+ return err
+ }
+
+ projectMod := createProjectModule(c.gomods.GetMain(), c.workingDir, c.moduleConfig)
+
+ if err := c.addAndRecurse(projectMod); err != nil {
+ return err
+ }
+
+ // Append the project module at the tail.
+ c.modules = append(c.modules, projectMod)
+
+ return nil
+}
+
+func (c *collector) collectModulesTXT(owner Module) error {
+ vendorDir := filepath.Join(owner.Dir(), vendord)
+ filename := filepath.Join(vendorDir, vendorModulesFilename)
+
+ f, err := c.fs.Open(filename)
+
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil
+ }
+
+ return err
+ }
+
+ defer f.Close()
+
+ scanner := bufio.NewScanner(f)
+
+ for scanner.Scan() {
+ // # github.com/alecthomas/chroma v0.6.3
+ line := scanner.Text()
+ line = strings.Trim(line, "# ")
+ line = strings.TrimSpace(line)
+ parts := strings.Fields(line)
+ if len(parts) != 2 {
+ return errors.Errorf("invalid modules list: %q", filename)
+ }
+ path := parts[0]
+ if _, found := c.vendored[path]; !found {
+ c.vendored[path] = vendoredModule{
+ Owner: owner,
+ Dir: filepath.Join(vendorDir, path),
+ Version: parts[1],
+ }
+ }
+
+ }
+ return nil
+}
+
+func (c *collector) loadModules() error {
+ modules, err := c.listGoMods()
+ if err != nil {
+ return err
+ }
+ c.gomods = modules
+ return nil
+}
+
+func (c *collector) wrapModuleNotFound(err error) error {
+ if c.GoModulesFilename == "" {
+ return err
+ }
+
+ baseMsg := "we found a go.mod file in your project, but"
+
+ switch c.goBinaryStatus {
+ case goBinaryStatusNotFound:
+ return errors.Wrap(err, baseMsg+" you need to install Go to use it. See https://golang.org/dl/.")
+ case goBinaryStatusTooOld:
+ return errors.Wrap(err, baseMsg+" you need to a newer version of Go to use it. See https://golang.org/dl/.")
+ }
+
+ return err
+
+}
+
+// In the first iteration of Hugo Modules, we do not support multiple
+// major versions running at the same time, so we pick the first (upper most).
+// We will investigate namespaces in future versions.
+// TODO(bep) mod add a warning when the above happens.
+func pathKey(p string) string {
+ prefix, _, _ := module.SplitPathVersion(p)
+
+ return strings.ToLower(prefix)
+}
diff --git a/modules/collect_test.go b/modules/collect_test.go
new file mode 100644
index 00000000000..d76c0b2bbed
--- /dev/null
+++ b/modules/collect_test.go
@@ -0,0 +1,38 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestPathKey(t *testing.T) {
+ assert := require.New(t)
+
+ for _, test := range []struct {
+ in string
+ expect string
+ }{
+ {"github.com/foo", "github.com/foo"},
+ {"github.com/foo/v2", "github.com/foo"},
+ {"github.com/foo/v12", "github.com/foo"},
+ {"github.com/foo/v3d", "github.com/foo/v3d"},
+ {"MyTheme", "mytheme"},
+ } {
+ assert.Equal(test.expect, pathKey(test.in))
+ }
+
+}
diff --git a/modules/config.go b/modules/config.go
new file mode 100644
index 00000000000..1a113ec5e9f
--- /dev/null
+++ b/modules/config.go
@@ -0,0 +1,251 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+ "fmt"
+ "path"
+ "path/filepath"
+ "strings"
+
+ "github.com/gohugoio/hugo/config"
+ "github.com/gohugoio/hugo/hugofs/files"
+ "github.com/gohugoio/hugo/langs"
+ "github.com/mitchellh/mapstructure"
+)
+
+type Config struct {
+ Mounts []Mount
+ Imports []Import
+}
+
+type Import struct {
+ Path string // Module path
+ Mounts []Mount
+}
+
+type Mount struct {
+ // TODO(bep) mod add IgnoreFiles: slice of GOB pattern matching content/*
+ Source string // relative path in source repo, e.g. "scss"
+ Target string // relative target path, e.g. "assets/bootstrap/scss"
+
+ // TODO(bep) mod
+ Lang string
+}
+
+func (m Mount) Component() string {
+ return strings.Split(m.Target, fileSeparator)[0]
+}
+
+// ApplyProjectConfigDefaults applies default/missing module configuration for
+// the main project.
+func ApplyProjectConfigDefaults(cfg config.Provider, mod Module) error {
+ moda := mod.(*moduleAdapter)
+
+ // TODO(bep) mod we need a way to check if contentDir etc. is really set.
+ // ... so remove the default settings for these.
+
+ // Map legacy directory config into the new module.
+ languages := cfg.Get("languagesSortedDefaultFirst").(langs.Languages)
+ isMultiHost := languages.IsMultihost()
+
+ // To bridge between old and new configuration format we need
+ // a way to make sure all of the core components are configured on
+ // the basic level.
+ componentsConfigured := make(map[string]bool)
+ for _, mnt := range moda.mounts {
+ componentsConfigured[mnt.Component()] = true
+ }
+
+ type dirKeyComponent struct {
+ key string
+ component string
+ multilingual bool
+ }
+
+ dirKeys := []dirKeyComponent{
+ {"contentDir", files.ComponentFolderContent, true},
+ {"dataDir", files.ComponentFolderData, false},
+ {"layoutDir", files.ComponentFolderLayouts, false},
+ {"i18nDir", files.ComponentFolderI18n, false},
+ {"archetypeDir", files.ComponentFolderArchetypes, false},
+ {"assetDir", files.ComponentFolderAssets, false},
+ {"resourceDir", files.ComponentFolderResources, false},
+ {"", files.ComponentFolderStatic, isMultiHost},
+ }
+
+ createMountsFor := func(d dirKeyComponent, cfg config.Provider) []Mount {
+ var lang string
+ if language, ok := cfg.(*langs.Language); ok {
+ lang = language.Lang
+ }
+
+ // Static mounts are a little special.
+ if d.component == files.ComponentFolderStatic {
+ var mounts []Mount
+ staticDirs := getStaticDirs(cfg)
+ if len(staticDirs) > 0 {
+ componentsConfigured[d.component] = true
+ }
+
+ for _, dir := range staticDirs {
+ mounts = append(mounts, Mount{Lang: lang, Source: dir, Target: d.component})
+ }
+
+ return mounts
+
+ }
+
+ if cfg.IsSet(d.key) {
+ source := cfg.GetString(d.key)
+ componentsConfigured[d.component] = true
+
+ return []Mount{Mount{
+ // No lang set for layouts etc.
+ Source: source,
+ Target: d.component}}
+ }
+
+ return nil
+ }
+
+ createMounts := func(d dirKeyComponent) []Mount {
+ var mounts []Mount
+ if d.multilingual {
+ if d.component == files.ComponentFolderContent {
+ seen := make(map[string]bool)
+ for _, language := range languages {
+ contentDir := language.ContentDir
+ if seen[contentDir] {
+ continue
+ }
+ seen[contentDir] = true
+ mounts = append(mounts, Mount{Lang: language.Lang, Source: contentDir, Target: d.component})
+ }
+
+ componentsConfigured[d.component] = len(seen) > 0
+
+ } else {
+ for _, language := range languages {
+ mounts = append(mounts, createMountsFor(d, language)...)
+ }
+ }
+ } else {
+ mounts = append(mounts, createMountsFor(d, cfg)...)
+ }
+
+ return mounts
+ }
+
+ var mounts []Mount
+ for _, dirKey := range dirKeys {
+ if componentsConfigured[dirKey.component] {
+
+ continue
+ }
+
+ mounts = append(mounts, createMounts(dirKey)...)
+
+ }
+
+ // Add default configuration
+ for _, dirKey := range dirKeys {
+ if componentsConfigured[dirKey.component] {
+ continue
+ }
+ mounts = append(mounts, Mount{Source: dirKey.component, Target: dirKey.component})
+ }
+
+ // Remove duplicates
+ seen := make(map[string]bool)
+ tmp := mounts[:0]
+ for _, m := range mounts {
+ key := path.Join(m.Lang, m.Source, m.Target)
+ if !seen[key] {
+ tmp = append(tmp, m)
+ }
+ seen[key] = true
+ }
+
+ moda.mounts = mounts
+
+ return nil
+}
+
+// DecodeConfig creates a modules Config from a given Hugo configuration.
+func DecodeConfig(cfg config.Provider) (c Config, err error) {
+ if cfg == nil {
+ return
+ }
+
+ themeSet := cfg.IsSet("theme")
+ moduleSet := cfg.IsSet("module")
+
+ if moduleSet {
+ m := cfg.GetStringMap("module")
+ err = mapstructure.WeakDecode(m, &c)
+
+ for i, mnt := range c.Mounts {
+ mnt.Source = filepath.Clean(mnt.Source)
+ mnt.Target = filepath.Clean(mnt.Target)
+ c.Mounts[i] = mnt
+ }
+
+ }
+
+ if themeSet {
+ imports := config.GetStringSlicePreserveString(cfg, "theme")
+ for _, imp := range imports {
+ c.Imports = append(c.Imports, Import{
+ Path: imp,
+ })
+ }
+
+ }
+
+ return
+}
+
+func removeDuplicatesKeepRight(in []string) []string {
+ seen := make(map[string]bool)
+ var out []string
+ for i := len(in) - 1; i >= 0; i-- {
+ v := in[i]
+ if seen[v] {
+ continue
+ }
+ out = append([]string{v}, out...)
+ seen[v] = true
+ }
+
+ return out
+}
+
+func getStaticDirs(cfg config.Provider) []string {
+ var staticDirs []string
+ for i := -1; i <= 10; i++ {
+ staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...)
+ }
+ return staticDirs
+}
+
+func getStringOrStringSlice(cfg config.Provider, key string, id int) []string {
+
+ if id >= 0 {
+ key = fmt.Sprintf("%s%d", key, id)
+ }
+
+ return config.GetStringSlicePreserveString(cfg, key)
+
+}
diff --git a/modules/config_test.go b/modules/config_test.go
new file mode 100644
index 00000000000..a372c39b1fc
--- /dev/null
+++ b/modules/config_test.go
@@ -0,0 +1,95 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+ "testing"
+
+ "github.com/gohugoio/hugo/config"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestDecodeConfig(t *testing.T) {
+ assert := require.New(t)
+ tomlConfig := `
+[module]
+[[module.mounts]]
+source="src/project/blog"
+target="content/blog"
+lang="en"
+[[module.imports]]
+path="github.com/bep/mycomponent"
+[[module.imports.mounts]]
+source="scss"
+target="assets/bootstrap/scss"
+[[module.imports.mounts]]
+source="src/markdown/blog"
+target="content/blog"
+lang="en"
+`
+ cfg, err := config.FromConfigString(tomlConfig, "toml")
+ assert.NoError(err)
+
+ mcfg, err := DecodeConfig(cfg)
+ assert.NoError(err)
+
+ assert.Len(mcfg.Mounts, 1)
+ assert.Len(mcfg.Imports, 1)
+ imp := mcfg.Imports[0]
+ imp.Path = "github.com/bep/mycomponent"
+ assert.Equal("src/markdown/blog", imp.Mounts[1].Source)
+ assert.Equal("content/blog", imp.Mounts[1].Target)
+ assert.Equal("en", imp.Mounts[1].Lang)
+
+}
+
+// Test old style theme import.
+func TestDecodeConfigTheme(t *testing.T) {
+ assert := require.New(t)
+ tomlConfig := `
+
+theme = ["a", "b"]
+`
+ cfg, err := config.FromConfigString(tomlConfig, "toml")
+ assert.NoError(err)
+
+ mcfg, err := DecodeConfig(cfg)
+ assert.NoError(err)
+
+ assert.Len(mcfg.Imports, 2)
+ assert.Equal("a", mcfg.Imports[0].Path)
+ assert.Equal("b", mcfg.Imports[1].Path)
+}
+
+func TestDecodeConfigBothOldAndNewProvided(t *testing.T) {
+ assert := require.New(t)
+ tomlConfig := `
+
+theme = ["b", "c"]
+
+[module]
+[[module.imports]]
+path="a"
+
+`
+ cfg, err := config.FromConfigString(tomlConfig, "toml")
+ assert.NoError(err)
+
+ modCfg, err := DecodeConfig(cfg)
+ assert.NoError(err)
+ assert.Len(modCfg.Imports, 3)
+ assert.Equal("a", modCfg.Imports[0].Path)
+
+}
diff --git a/modules/module.go b/modules/module.go
new file mode 100644
index 00000000000..08258135541
--- /dev/null
+++ b/modules/module.go
@@ -0,0 +1,166 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package modules provides a client that can be used to manage Hugo Components,
+// what's refered to as Hugo Modules. Hugo Modules is built on top of Go Modules,
+// but also supports vendoring and components stored directly in the themes dir.
+package modules
+
+import (
+ "github.com/gohugoio/hugo/config"
+ "github.com/spf13/afero"
+)
+
+var _ Module = (*moduleAdapter)(nil)
+
+type Module interface {
+
+ // Optional configuration filename (e.g. "/themes/mytheme/config.json").
+ // This will be added to the special configuration watch list when in
+ // server mode.
+ ConfigFilename() string
+
+ // Optional config read from the configFilename above.
+ // TODO(bep) mod check usage
+ Cfg() config.Provider
+
+ Config() Config
+
+ // Directory holding files for this module.
+ Dir() string
+
+ // Returns whether this is a Go Module.
+ IsGoMod() bool
+
+ // In the dependency tree, this is the first module that defines this module
+ // as a dependency.
+ Owner() Module
+
+ // Replaced by this module.
+ Replace() Module
+
+ // Any directory remappings.
+ Mounts() []Mount
+
+ // Returns the path to this module.
+ // This will either be the module path, e.g. "github.com/gohugoio/myshortcodes",
+ // or the path below your /theme folder, e.g. "mytheme".
+ Path() string
+
+ // Returns whether Dir points below the _vendor dir.
+ Vendor() bool
+
+ // The module version.
+ Version() string
+}
+
+type Modules []Module
+
+type moduleAdapter struct {
+ path string
+ dir string
+ version string
+ vendor bool
+ projectMod bool
+ owner Module
+
+ mounts []Mount
+
+ configFilename string
+ cfg config.Provider
+ config Config
+
+ // Set if a Go module.
+ gomod *goModule
+}
+
+func (m *moduleAdapter) Cfg() config.Provider {
+ return m.cfg
+}
+
+func (m *moduleAdapter) Config() Config {
+ return m.config
+}
+
+func (m *moduleAdapter) ConfigFilename() string {
+ return m.configFilename
+}
+
+func (m *moduleAdapter) Dir() string {
+ // This may point to the _vendor dir.
+ if !m.IsGoMod() || m.dir != "" {
+ return m.dir
+ }
+ return m.gomod.Dir
+}
+
+func (m *moduleAdapter) IsGoMod() bool {
+ return m.gomod != nil
+}
+
+func (m *moduleAdapter) Owner() Module {
+ return m.owner
+}
+
+func (m *moduleAdapter) Replace() Module {
+ if m.IsGoMod() && !m.Vendor() && m.gomod.Replace != nil {
+ return &moduleAdapter{
+ gomod: m.gomod.Replace,
+ owner: m.owner,
+ }
+ }
+ return nil
+}
+
+func (m *moduleAdapter) Mounts() []Mount {
+ return m.mounts
+}
+
+func (m *moduleAdapter) Path() string {
+ if !m.IsGoMod() || m.path != "" {
+ return m.path
+ }
+ return m.gomod.Path
+}
+
+func (m *moduleAdapter) Vendor() bool {
+ return m.vendor
+}
+
+func (m *moduleAdapter) Version() string {
+ if !m.IsGoMod() || m.version != "" {
+ return m.version
+ }
+ return m.gomod.Version
+}
+
+func (m *moduleAdapter) validateAndApplyDefaults(fs afero.Fs) error {
+
+ /*if len(m.modImport.Mounts) == 0 {
+ // Create default mount points for every component folder that
+ // exists in the module.
+ for _, componentFolder := range files.ComponentFolders {
+ sourceDir := filepath.Join(dir, componentFolder)
+ _, err := fs.Stat(sourceDir)
+ if err == nil {
+ m.modImport.Mounts = append(m.modImport.Mounts, Mount{
+ Source: componentFolder,
+ Target: componentFolder,
+ })
+ }
+ }
+ }*/
+
+ return nil
+
+}
diff --git a/resources/image_cache.go b/resources/image_cache.go
index cf1e999badc..4072851e29d 100644
--- a/resources/image_cache.go
+++ b/resources/image_cache.go
@@ -145,7 +145,7 @@ func (c *imageCache) getOrCreate(
}
// The file is now stored in this cache.
- img.overriddenSourceFs = c.fileCache.Fs
+ img.sourceFs = c.fileCache.Fs
c.mu.Lock()
if img2, found := c.store[key]; found {
diff --git a/resources/page/page_nop.go b/resources/page/page_nop.go
index 229bcb077e4..c3a4819f1f4 100644
--- a/resources/page/page_nop.go
+++ b/resources/page/page_nop.go
@@ -17,9 +17,10 @@ package page
import (
"html/template"
- "os"
"time"
+ "github.com/gohugoio/hugo/hugofs"
+
"github.com/bep/gitmap"
"github.com/gohugoio/hugo/navigation"
@@ -147,7 +148,7 @@ func (p *nopPage) File() source.File {
return nilFile
}
-func (p *nopPage) FileInfo() os.FileInfo {
+func (p *nopPage) FileInfo() hugofs.FileMetaInfo {
return nil
}
diff --git a/resources/page/page_wrappers.autogen.go b/resources/page/page_wrappers.autogen.go
index d7fcb520109..d2d14dee6f1 100644
--- a/resources/page/page_wrappers.autogen.go
+++ b/resources/page/page_wrappers.autogen.go
@@ -18,8 +18,8 @@ package page
import (
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/helpers"
+ "github.com/gohugoio/hugo/hugofs"
"html/template"
- "os"
)
// NewDeprecatedWarningPage adds deprecation warnings to the given implementation.
@@ -91,7 +91,7 @@ func (p *pageDeprecated) UniqueID() string {
helpers.Deprecated("Page", ".UniqueID", "Use .File.UniqueID", false)
return p.p.UniqueID()
}
-func (p *pageDeprecated) FileInfo() os.FileInfo {
+func (p *pageDeprecated) FileInfo() hugofs.FileMetaInfo {
helpers.Deprecated("Page", ".FileInfo", "Use .File.FileInfo", false)
return p.p.FileInfo()
}
diff --git a/resources/page/permalinks.go b/resources/page/permalinks.go
index 98489231b2c..d0dc3a8f21e 100644
--- a/resources/page/permalinks.go
+++ b/resources/page/permalinks.go
@@ -91,6 +91,7 @@ func (l PermalinkExpander) parse(patterns map[string]string) (map[string]func(Pa
expanders := make(map[string]func(Page) (string, error))
for k, pattern := range patterns {
+ k = strings.TrimSpace(k)
if !l.validate(pattern) {
return nil, &permalinkExpandError{pattern: pattern, err: errPermalinkIllFormed}
}
diff --git a/resources/page/site.go b/resources/page/site.go
index 25df063f1b1..9153c855614 100644
--- a/resources/page/site.go
+++ b/resources/page/site.go
@@ -17,6 +17,8 @@ import (
"html/template"
"time"
+ "github.com/gohugoio/hugo/config"
+
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/langs"
"github.com/gohugoio/hugo/navigation"
@@ -51,3 +53,72 @@ func (s Sites) First() Site {
}
return s[0]
}
+
+type testSite struct {
+ h hugo.Info
+ l *langs.Language
+}
+
+func (t testSite) Hugo() hugo.Info {
+ return t.h
+}
+
+func (t testSite) ServerPort() int {
+ return 1313
+}
+
+func (testSite) LastChange() (t time.Time) {
+ return
+}
+
+func (t testSite) Title() string {
+ return "foo"
+}
+
+func (t testSite) Sites() Sites {
+ return nil
+}
+
+func (t testSite) IsServer() bool {
+ return false
+}
+
+func (t testSite) Language() *langs.Language {
+ return t.l
+}
+
+func (t testSite) Pages() Pages {
+ return nil
+}
+
+func (t testSite) RegularPages() Pages {
+ return nil
+}
+
+func (t testSite) Menus() navigation.Menus {
+ return nil
+}
+
+func (t testSite) Taxonomies() interface{} {
+ return nil
+}
+
+func (t testSite) BaseURL() template.URL {
+ return ""
+}
+
+func (t testSite) Params() map[string]interface{} {
+ return nil
+}
+
+func (t testSite) Data() map[string]interface{} {
+ return nil
+}
+
+// NewDummyHugoSite creates a new minimal test site.
+func NewDummyHugoSite(cfg config.Provider) Site {
+ return testSite{
+ h: hugo.NewInfo(hugo.EnvironmentProduction),
+ l: langs.NewLanguage("en", cfg),
+ }
+}
diff --git a/resources/page/testhelpers_test.go b/resources/page/testhelpers_test.go
index 60a6c0816fb..fa5f8e9c8a0 100644
--- a/resources/page/testhelpers_test.go
+++ b/resources/page/testhelpers_test.go
@@ -16,10 +16,11 @@ package page
import (
"fmt"
"html/template"
- "os"
"path/filepath"
"time"
+ "github.com/gohugoio/hugo/modules"
+
"github.com/bep/gitmap"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/resources/resource"
@@ -65,6 +66,12 @@ func newTestPathSpec() *helpers.PathSpec {
func newTestPathSpecFor(cfg config.Provider) *helpers.PathSpec {
config.SetBaseTestDefaults(cfg)
+ langs.LoadLanguageSettings(cfg, nil)
+ mod, err := modules.CreateProjectModule(cfg)
+ if err != nil {
+ panic(err)
+ }
+ cfg.Set("allModules", modules.Modules{mod})
fs := hugofs.NewMem(cfg)
s, err := helpers.NewPathSpec(fs, cfg)
if err != nil {
@@ -189,7 +196,7 @@ func (p *testPage) File() source.File {
return p.file
}
-func (p *testPage) FileInfo() os.FileInfo {
+func (p *testPage) FileInfo() hugofs.FileMetaInfo {
panic("not implemented")
}
diff --git a/resources/page/zero_file.autogen.go b/resources/page/zero_file.autogen.go
index eec1dd66dc0..23e36b76490 100644
--- a/resources/page/zero_file.autogen.go
+++ b/resources/page/zero_file.autogen.go
@@ -17,8 +17,8 @@ package page
import (
"github.com/gohugoio/hugo/helpers"
+ "github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/source"
- "os"
)
// ZeroFile represents a zero value of source.File with warnings if invoked.
@@ -82,7 +82,7 @@ func (z zeroFile) UniqueID() (o0 string) {
z.log.Println(".File.UniqueID on zero object. Wrap it in if or with: {{ with .File }}{{ .UniqueID }}{{ end }}")
return
}
-func (z zeroFile) FileInfo() (o0 os.FileInfo) {
+func (z zeroFile) FileInfo() (o0 hugofs.FileMetaInfo) {
z.log.Println(".File.FileInfo on zero object. Wrap it in if or with: {{ with .File }}{{ .FileInfo }}{{ end }}")
return
}
diff --git a/resources/resource.go b/resources/resource.go
index c120a8dd090..d871f2bc1f0 100644
--- a/resources/resource.go
+++ b/resources/resource.go
@@ -24,6 +24,8 @@ import (
"strings"
"sync"
+ "github.com/gohugoio/hugo/hugofs"
+
"github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/output"
@@ -133,9 +135,13 @@ type ResourceSourceDescriptor struct {
SourceFile source.File
OpenReadSeekCloser resource.OpenReadSeekCloser
+ FileInfo os.FileInfo
+
// If OpenReadSeekerCloser is not set, we use this to open the file.
SourceFilename string
+ Fs afero.Fs
+
// The relative target filename without any language code.
RelTargetFilename string
@@ -157,19 +163,11 @@ func (r ResourceSourceDescriptor) Filename() string {
return r.SourceFilename
}
-func (r *Spec) sourceFs() afero.Fs {
- return r.PathSpec.BaseFs.Content.Fs
-}
-
func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) {
- return r.newResourceForFs(r.sourceFs(), fd)
-}
-
-func (r *Spec) NewForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) {
- return r.newResourceForFs(sourceFs, fd)
+ return r.newResourceFor(fd)
}
-func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) {
+func (r *Spec) newResourceFor(fd ResourceSourceDescriptor) (resource.Resource, error) {
if fd.OpenReadSeekCloser == nil {
if fd.SourceFile != nil && fd.SourceFilename != "" {
return nil, errors.New("both SourceFile and AbsSourceFilename provided")
@@ -187,15 +185,14 @@ func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor)
fd.TargetBasePaths = r.MultihostTargetBasePaths
}
- return r.newResource(sourceFs, fd)
+ return r.newResource(fd.Fs, fd)
}
func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) {
- var fi os.FileInfo
+ fi := fd.FileInfo
var sourceFilename string
if fd.OpenReadSeekCloser != nil {
-
} else if fd.SourceFilename != "" {
var err error
fi, err = sourceFs.Stat(fd.SourceFilename)
@@ -207,7 +204,6 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso
}
sourceFilename = fd.SourceFilename
} else {
- fi = fd.SourceFile.FileInfo()
sourceFilename = fd.SourceFile.Filename()
}
@@ -245,8 +241,6 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso
mimeType)
if mimeType.MainType == "image" {
- ext := strings.ToLower(helpers.Ext(sourceFilename))
-
imgFormat, ok := imageFormats[ext]
if !ok {
// This allows SVG etc. to be used as resources. They will not have the methods of the Image, but
@@ -376,7 +370,7 @@ type genericResource struct {
// This may be set to tell us to look in another filesystem for this resource.
// We, by default, use the sourceFs filesystem in the spec below.
- overriddenSourceFs afero.Fs
+ sourceFs afero.Fs
spec *Spec
@@ -411,7 +405,11 @@ func (l *genericResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
if l.openReadSeekerCloser != nil {
return l.openReadSeekerCloser()
}
- f, err := l.sourceFs().Open(l.sourceFilename)
+ if fim, ok := l.osFileInfo.(hugofs.FileMetaInfo); ok {
+ return fim.Meta().Open()
+ }
+ // TODO(bep) mod
+ f, err := l.getSourceFs().Open(l.sourceFilename)
if err != nil {
return nil, err
}
@@ -497,11 +495,8 @@ func (l *genericResource) initContent() error {
return err
}
-func (l *genericResource) sourceFs() afero.Fs {
- if l.overriddenSourceFs != nil {
- return l.overriddenSourceFs
- }
- return l.spec.sourceFs()
+func (l *genericResource) getSourceFs() afero.Fs {
+ return l.sourceFs
}
func (l *genericResource) publishIfNeeded() {
@@ -711,6 +706,10 @@ func (r *Spec) newGenericResourceWithBase(
baseFilename string,
mediaType media.Type) *genericResource {
+ if osFileInfo != nil && osFileInfo.IsDir() {
+ panic(fmt.Sprintf("dirs nto supported resource types: %v", osFileInfo))
+ }
+
// This value is used both to construct URLs and file paths, but start
// with a Unix-styled path.
baseFilename = helpers.ToSlashTrimLeading(baseFilename)
@@ -738,7 +737,7 @@ func (r *Spec) newGenericResourceWithBase(
openReadSeekerCloser: openReadSeekerCloser,
publishOnce: po,
resourcePathDescriptor: pathDescriptor,
- overriddenSourceFs: sourceFs,
+ sourceFs: sourceFs,
osFileInfo: osFileInfo,
sourceFilename: sourceFilename,
mediaType: mediaType,
diff --git a/resources/resource_factories/bundler/bundler.go b/resources/resource_factories/bundler/bundler.go
index 59810e34741..6655ee5c330 100644
--- a/resources/resource_factories/bundler/bundler.go
+++ b/resources/resource_factories/bundler/bundler.go
@@ -124,9 +124,9 @@ func (c *Client) Concat(targetPath string, r resource.Resources) (resource.Resou
return &multiReadSeekCloser{mr: mr, sources: rcsources}, nil
}
- composite, err := c.rs.NewForFs(
- c.rs.FileCaches.AssetsCache().Fs,
+ composite, err := c.rs.New(
resources.ResourceSourceDescriptor{
+ Fs: c.rs.FileCaches.AssetsCache().Fs,
LazyPublish: true,
OpenReadSeekCloser: concatr,
RelTargetFilename: filepath.Clean(targetPath)})
diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go
index dc565056d8e..36a29e733fb 100644
--- a/resources/resource_factories/create/create.go
+++ b/resources/resource_factories/create/create.go
@@ -40,10 +40,10 @@ func New(rs *resources.Spec) *Client {
func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) {
filename = filepath.Clean(filename)
return c.rs.ResourceCache.GetOrCreate(resources.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) {
- return c.rs.NewForFs(fs,
- resources.ResourceSourceDescriptor{
- LazyPublish: true,
- SourceFilename: filename})
+ return c.rs.New(resources.ResourceSourceDescriptor{
+ Fs: fs,
+ LazyPublish: true,
+ SourceFilename: filename})
})
}
@@ -51,9 +51,9 @@ func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) {
// FromString creates a new Resource from a string with the given relative target path.
func (c *Client) FromString(targetPath, content string) (resource.Resource, error) {
return c.rs.ResourceCache.GetOrCreate(resources.CACHE_OTHER, targetPath, func() (resource.Resource, error) {
- return c.rs.NewForFs(
- c.rs.FileCaches.AssetsCache().Fs,
+ return c.rs.New(
resources.ResourceSourceDescriptor{
+ Fs: c.rs.FileCaches.AssetsCache().Fs,
LazyPublish: true,
OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
return hugio.NewReadSeekerNoOpCloserFromString(content), nil
diff --git a/resources/resource_test.go b/resources/resource_test.go
index af7867eb1c9..565ae06c433 100644
--- a/resources/resource_test.go
+++ b/resources/resource_test.go
@@ -21,6 +21,8 @@ import (
"testing"
"time"
+ "github.com/spf13/afero"
+
"github.com/gohugoio/hugo/resources/resource"
"github.com/gohugoio/hugo/media"
@@ -61,7 +63,9 @@ func TestNewResourceFromFilename(t *testing.T) {
writeSource(t, spec.Fs, "content/a/b/logo.png", "image")
writeSource(t, spec.Fs, "content/a/b/data.json", "json")
- r, err := spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/logo.png"})
+ bfs := afero.NewBasePathFs(spec.Fs.Source, "content")
+
+ r, err := spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: "a/b/logo.png"})
assert.NoError(err)
assert.NotNil(r)
@@ -69,7 +73,7 @@ func TestNewResourceFromFilename(t *testing.T) {
assert.Equal("/a/b/logo.png", r.RelPermalink())
assert.Equal("https://example.com/a/b/logo.png", r.Permalink())
- r, err = spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/data.json"})
+ r, err = spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: "a/b/data.json"})
assert.NoError(err)
assert.NotNil(r)
@@ -85,8 +89,10 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) {
spec := newTestResourceSpecForBaseURL(assert, "https://example.com/docs")
writeSource(t, spec.Fs, "content/a/b/logo.png", "image")
+ bfs := afero.NewBasePathFs(spec.Fs.Source, "content")
- r, err := spec.New(ResourceSourceDescriptor{SourceFilename: filepath.FromSlash("a/b/logo.png")})
+ fmt.Println()
+ r, err := spec.New(ResourceSourceDescriptor{Fs: bfs, SourceFilename: filepath.FromSlash("a/b/logo.png")})
assert.NoError(err)
assert.NotNil(r)
diff --git a/resources/resource_transformers/postcss/postcss.go b/resources/resource_transformers/postcss/postcss.go
index 5350eebc5b6..28b480cb83d 100644
--- a/resources/resource_transformers/postcss/postcss.go
+++ b/resources/resource_transformers/postcss/postcss.go
@@ -130,7 +130,7 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC
if !filepath.IsAbs(configFile) {
// We resolve this against the virtual Work filesystem, to allow
// this config file to live in one of the themes if needed.
- fi, err := t.rs.BaseFs.Work.Fs.Stat(configFile)
+ fi, err := t.rs.BaseFs.Work.Stat(configFile)
if err != nil {
if t.options.Config != "" {
// Only fail if the user specificed config file is not found.
@@ -138,7 +138,7 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC
}
configFile = ""
} else {
- configFile = fi.(hugofs.RealFilenameInfo).RealFilename()
+ configFile = fi.(hugofs.FileMetaInfo).Meta().Filename()
}
}
diff --git a/resources/resource_transformers/tocss/scss/client.go b/resources/resource_transformers/tocss/scss/client.go
index 41ff6743355..e69af2f748f 100644
--- a/resources/resource_transformers/tocss/scss/client.go
+++ b/resources/resource_transformers/tocss/scss/client.go
@@ -19,6 +19,7 @@ import (
"github.com/gohugoio/hugo/hugolib/filesystems"
"github.com/gohugoio/hugo/resources"
"github.com/gohugoio/hugo/resources/resource"
+ "github.com/spf13/afero"
"github.com/mitchellh/mapstructure"
)
@@ -26,7 +27,7 @@ import (
type Client struct {
rs *resources.Spec
sfs *filesystems.SourceFilesystem
- workFs *filesystems.SourceFilesystem
+ workFs afero.Fs
}
func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) (*Client, error) {
diff --git a/resources/resource_transformers/tocss/scss/tocss.go b/resources/resource_transformers/tocss/scss/tocss.go
index 17c32ea8ece..ad581d681c6 100644
--- a/resources/resource_transformers/tocss/scss/tocss.go
+++ b/resources/resource_transformers/tocss/scss/tocss.go
@@ -55,7 +55,11 @@ func (t *toCSSTransformation) Transform(ctx *resources.ResourceTransformationCtx
// Append any workDir relative include paths
for _, ip := range options.from.IncludePaths {
- options.to.IncludePaths = append(options.to.IncludePaths, t.c.workFs.RealDirs(filepath.Clean(ip))...)
+ info, err := t.c.workFs.Stat(filepath.Clean(ip))
+ if err == nil {
+ filename := info.(hugofs.FileMetaInfo).Meta().Filename()
+ options.to.IncludePaths = append(options.to.IncludePaths, filename)
+ }
}
// To allow for overrides of SCSS files anywhere in the project/theme hierarchy, we need
@@ -74,6 +78,7 @@ func (t *toCSSTransformation) Transform(ctx *resources.ResourceTransformationCtx
prevDir = baseDir
} else {
prevDir = t.c.sfs.MakePathRelative(filepath.Dir(prev))
+
if prevDir == "" {
// Not a member of this filesystem. Let LibSASS handle it.
return "", "", false
@@ -100,8 +105,8 @@ func (t *toCSSTransformation) Transform(ctx *resources.ResourceTransformationCtx
filenameToCheck := filepath.Join(basePath, fmt.Sprintf(namePattern, name))
fi, err := t.c.sfs.Fs.Stat(filenameToCheck)
if err == nil {
- if fir, ok := fi.(hugofs.RealFilenameInfo); ok {
- return fir.RealFilename(), "", true
+ if fim, ok := fi.(hugofs.FileMetaInfo); ok {
+ return fim.Meta().Filename(), "", true
}
}
}
diff --git a/resources/sunset.jpg b/resources/sunset.jpg
new file mode 100644
index 00000000000..7d7307bed36
Binary files /dev/null and b/resources/sunset.jpg differ
diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go
index d064fa57052..61a9a6e1e72 100644
--- a/resources/testhelpers_test.go
+++ b/resources/testhelpers_test.go
@@ -4,7 +4,6 @@ import (
"path/filepath"
"testing"
- "fmt"
"image"
"io"
"io/ioutil"
@@ -12,6 +11,9 @@ import (
"runtime"
"strings"
+ "github.com/gohugoio/hugo/langs"
+ "github.com/gohugoio/hugo/modules"
+
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
@@ -28,9 +30,8 @@ func newTestResourceSpec(assert *require.Assertions) *Spec {
return newTestResourceSpecForBaseURL(assert, "https://example.com/")
}
-func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *Spec {
+func createTestCfg() *viper.Viper {
cfg := viper.New()
- cfg.Set("baseURL", baseURL)
cfg.Set("resourceDir", "resources")
cfg.Set("contentDir", "content")
cfg.Set("dataDir", "data")
@@ -40,6 +41,21 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *
cfg.Set("archetypeDir", "archetypes")
cfg.Set("publishDir", "public")
+ langs.LoadLanguageSettings(cfg, nil)
+ mod, err := modules.CreateProjectModule(cfg)
+ if err != nil {
+ panic(err)
+ }
+ cfg.Set("allModules", modules.Modules{mod})
+
+ return cfg
+
+}
+
+func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *Spec {
+ cfg := createTestCfg()
+ cfg.Set("baseURL", baseURL)
+
imagingCfg := map[string]interface{}{
"resampleFilter": "linear",
"quality": 68,
@@ -71,7 +87,7 @@ func newTargetPaths(link string) func() page.TargetPaths {
}
func newTestResourceOsFs(assert *require.Assertions) *Spec {
- cfg := viper.New()
+ cfg := createTestCfg()
cfg.Set("baseURL", "https://example.com")
workDir, _ := ioutil.TempDir("", "hugores")
@@ -83,14 +99,6 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec {
}
cfg.Set("workingDir", workDir)
- cfg.Set("resourceDir", "resources")
- cfg.Set("contentDir", "content")
- cfg.Set("dataDir", "data")
- cfg.Set("i18nDir", "i18n")
- cfg.Set("layoutDir", "layouts")
- cfg.Set("assetDir", "assets")
- cfg.Set("archetypeDir", "archetypes")
- cfg.Set("publishDir", "public")
fs := hugofs.NewFrom(hugofs.Os, cfg)
fs.Destination = &afero.MemMapFs{}
@@ -126,7 +134,7 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) r
src, err := os.Open(filepath.FromSlash("testdata/" + name))
assert.NoError(err)
- out, err := helpers.OpenFileForWriting(spec.BaseFs.Content.Fs, name)
+ out, err := helpers.OpenFileForWriting(spec.Fs.Source, name)
assert.NoError(err)
_, err = io.Copy(out, src)
out.Close()
@@ -135,7 +143,7 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) r
factory := newTargetPaths("/a")
- r, err := spec.New(ResourceSourceDescriptor{TargetPaths: factory, LazyPublish: true, SourceFilename: name})
+ r, err := spec.New(ResourceSourceDescriptor{Fs: spec.Fs.Source, TargetPaths: factory, LazyPublish: true, SourceFilename: name})
assert.NoError(err)
return r.(resource.ContentResource)
@@ -144,9 +152,6 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) r
func assertImageFile(assert *require.Assertions, fs afero.Fs, filename string, width, height int) {
filename = filepath.Clean(filename)
f, err := fs.Open(filename)
- if err != nil {
- printFs(fs, "", os.Stdout)
- }
assert.NoError(err)
defer f.Close()
@@ -170,22 +175,3 @@ func writeToFs(t testing.TB, fs afero.Fs, filename, content string) {
t.Fatalf("Failed to write file: %s", err)
}
}
-
-func printFs(fs afero.Fs, path string, w io.Writer) {
- if fs == nil {
- return
- }
- afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
- if info != nil && !info.IsDir() {
- s := path
- if lang, ok := info.(hugofs.LanguageAnnouncer); ok {
- s = s + "\t" + lang.Lang()
- }
- if fp, ok := info.(hugofs.FilePather); ok {
- s += "\tFilename: " + fp.Filename() + "\tBase: " + fp.BaseDir()
- }
- fmt.Fprintln(w, " ", s)
- }
- return nil
- })
-}
diff --git a/resources/transform.go b/resources/transform.go
index 934c713277b..b848556c61c 100644
--- a/resources/transform.go
+++ b/resources/transform.go
@@ -515,6 +515,9 @@ func (r *transformedResource) initTransform(setContent, publish bool) error {
// Copy the file from cache to /public
_, src, err := r.cache.fileCache.Get(r.sourceFilename)
+ if src == nil {
+ panic(fmt.Sprintf("[BUG] resource cache file not found: %q", r.sourceFilename))
+ }
if err == nil {
defer src.Close()
diff --git a/source/fileInfo.go b/source/fileInfo.go
index 072b55b6cbb..a4cbf6fe621 100644
--- a/source/fileInfo.go
+++ b/source/fileInfo.go
@@ -14,11 +14,14 @@
package source
import (
- "os"
"path/filepath"
"strings"
"sync"
+ "github.com/gohugoio/hugo/hugofs/files"
+
+ "github.com/pkg/errors"
+
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/hugofs"
@@ -28,8 +31,7 @@ import (
// fileInfo implements the File interface.
var (
- _ File = (*FileInfo)(nil)
- _ ReadableFile = (*FileInfo)(nil)
+ _ File = (*FileInfo)(nil)
)
// File represents a source file.
@@ -90,13 +92,7 @@ type FileWithoutOverlap interface {
// Hugo content files being one of them, considered to be unique.
UniqueID() string
- FileInfo() os.FileInfo
-}
-
-// A ReadableFile is a File that is readable.
-type ReadableFile interface {
- File
- Open() (hugio.ReadSeekCloser, error)
+ FileInfo() hugofs.FileMetaInfo
}
// FileInfo describes a source file.
@@ -107,7 +103,7 @@ type FileInfo struct {
sp *SourceSpec
- fi os.FileInfo
+ fi hugofs.FileMetaInfo
// Derived from filename
ext string // Extension without any "."
@@ -179,13 +175,14 @@ func (fi *FileInfo) UniqueID() string {
}
// FileInfo returns a file's underlying os.FileInfo.
-func (fi *FileInfo) FileInfo() os.FileInfo { return fi.fi }
+func (fi *FileInfo) FileInfo() hugofs.FileMetaInfo { return fi.fi }
func (fi *FileInfo) String() string { return fi.BaseFileName() }
// Open implements ReadableFile.
func (fi *FileInfo) Open() (hugio.ReadSeekCloser, error) {
- f, err := fi.sp.SourceFs.Open(fi.Filename())
+ f, err := fi.fi.Meta().Open()
+
return f, err
}
@@ -225,38 +222,45 @@ func NewTestFile(filename string) *FileInfo {
}
}
-// NewFileInfo returns a new FileInfo structure.
-func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, fi os.FileInfo) *FileInfo {
+func (sp *SourceSpec) NewFileInfoFrom(path, filename string) (*FileInfo, error) {
+ meta := hugofs.FileMeta{
+ "filename": filename,
+ "path": path,
+ }
- var lang, translationBaseName, relPath string
+ return sp.NewFileInfo(hugofs.NewFileMetaInfo(nil, meta))
+}
- if fp, ok := fi.(hugofs.FilePather); ok {
- filename = fp.Filename()
- baseDir = fp.BaseDir()
- relPath = fp.Path()
- }
+func (sp *SourceSpec) NewFileInfo(fi hugofs.FileMetaInfo) (*FileInfo, error) {
- if fl, ok := fi.(hugofs.LanguageAnnouncer); ok {
- lang = fl.Lang()
- translationBaseName = fl.TranslationBaseName()
- }
+ m := fi.Meta()
- dir, name := filepath.Split(filename)
- if !strings.HasSuffix(dir, helpers.FilePathSeparator) {
- dir = dir + helpers.FilePathSeparator
+ filename := m.Filename()
+ relPath := m.Path()
+ isLeafBundle := m.Classifier() == files.ContentClassLeaf
+
+ if relPath == "" || strings.Contains(relPath, "TODO") {
+ return nil, errors.Errorf("no Path provided by %v (%T)", m, m.Fs())
}
- baseDir = strings.TrimSuffix(baseDir, helpers.FilePathSeparator)
+ if filename == "" || strings.Contains(filename, "TODO") {
+ return nil, errors.Errorf("no Filename provided by %v (%T)", m, m.Fs())
+ }
- relDir := ""
- if dir != baseDir {
- relDir = strings.TrimPrefix(dir, baseDir)
+ relDir := filepath.Dir(relPath)
+ if relDir == "." {
+ relDir = ""
+ }
+ if !strings.HasSuffix(relDir, helpers.FilePathSeparator) {
+ relDir = relDir + helpers.FilePathSeparator
}
- relDir = strings.TrimPrefix(relDir, helpers.FilePathSeparator)
+ lang := m.Lang()
+ translationBaseName := m.GetString("translationBaseName")
- if relPath == "" {
- relPath = filepath.Join(relDir, name)
+ dir, name := filepath.Split(relPath)
+ if !strings.HasSuffix(dir, helpers.FilePathSeparator) {
+ dir = dir + helpers.FilePathSeparator
}
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), "."))
@@ -277,14 +281,14 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f
lang: lang,
ext: ext,
dir: dir,
- relDir: relDir,
- relPath: relPath,
+ relDir: relDir, // Dir()
+ relPath: relPath, // Path()
name: name,
- baseName: baseName,
+ baseName: baseName, // BaseFileName()
translationBaseName: translationBaseName,
isLeafBundle: isLeafBundle,
}
- return f
+ return f, nil
}
diff --git a/source/fileInfo_test.go b/source/fileInfo_test.go
index 9390c624706..0c024de1882 100644
--- a/source/fileInfo_test.go
+++ b/source/fileInfo_test.go
@@ -15,12 +15,9 @@ package source
import (
"path/filepath"
+ "strings"
"testing"
- "github.com/gohugoio/hugo/helpers"
-
- "github.com/gohugoio/hugo/hugofs"
- "github.com/spf13/afero"
"github.com/stretchr/testify/require"
)
@@ -55,56 +52,10 @@ func TestFileInfo(t *testing.T) {
}},
} {
- f := s.NewFileInfo(this.base, this.filename, false, nil)
+ path := strings.TrimPrefix(this.filename, this.base)
+ f, err := s.NewFileInfoFrom(path, this.filename)
+ assert.NoError(err)
this.assert(f)
}
}
-
-func TestFileInfoLanguage(t *testing.T) {
- assert := require.New(t)
- langs := map[string]bool{
- "sv": true,
- "en": true,
- }
-
- m := afero.NewMemMapFs()
- lfs := hugofs.NewLanguageFs("sv", langs, m)
- v := newTestConfig()
-
- fs := hugofs.NewFrom(m, v)
-
- ps, err := helpers.NewPathSpec(fs, v)
- assert.NoError(err)
- s := SourceSpec{SourceFs: lfs, PathSpec: ps}
- s.Languages = map[string]interface{}{
- "en": true,
- }
-
- err = afero.WriteFile(lfs, "page.md", []byte("abc"), 0777)
- assert.NoError(err)
- err = afero.WriteFile(lfs, "page.en.md", []byte("abc"), 0777)
- assert.NoError(err)
-
- sv, _ := lfs.Stat("page.md")
- en, _ := lfs.Stat("page.en.md")
-
- fiSv := s.NewFileInfo("", "page.md", false, sv)
- fiEn := s.NewFileInfo("", "page.en.md", false, en)
-
- assert.Equal("sv", fiSv.Lang())
- assert.Equal("en", fiEn.Lang())
-
- // test contentBaseName implementation
- fi := s.NewFileInfo("", "2018-10-01-contentbasename.md", false, nil)
- assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName())
-
- fi = s.NewFileInfo("", "2018-10-01-contentbasename.en.md", false, nil)
- assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName())
-
- fi = s.NewFileInfo("", filepath.Join("2018-10-01-contentbasename", "index.en.md"), true, nil)
- assert.Equal("2018-10-01-contentbasename", fi.ContentBaseName())
-
- fi = s.NewFileInfo("", filepath.Join("2018-10-01-contentbasename", "_index.en.md"), false, nil)
- assert.Equal("_index", fi.ContentBaseName())
-}
diff --git a/source/filesystem.go b/source/filesystem.go
index 0c1a6ac7b9e..f3357750878 100644
--- a/source/filesystem.go
+++ b/source/filesystem.go
@@ -19,24 +19,24 @@ import (
"runtime"
"sync"
- "github.com/gohugoio/hugo/helpers"
+ "github.com/pkg/errors"
+
+ "github.com/gohugoio/hugo/hugofs"
+
jww "github.com/spf13/jwalterweatherman"
- "golang.org/x/text/unicode/norm"
)
// Filesystem represents a source filesystem.
type Filesystem struct {
- files []ReadableFile
- filesInit sync.Once
+ files []File
+ filesInit sync.Once
+ filesInitErr error
Base string
- SourceSpec
-}
+ fi hugofs.FileMetaInfo
-// Input describes a source input.
-type Input interface {
- Files() []ReadableFile
+ SourceSpec
}
// NewFilesystem returns a new filesytem for a given source spec.
@@ -44,63 +44,84 @@ func (sp SourceSpec) NewFilesystem(base string) *Filesystem {
return &Filesystem{SourceSpec: sp, Base: base}
}
+func (sp SourceSpec) NewFilesystemFromFileMetaInfo(fi hugofs.FileMetaInfo) *Filesystem {
+ return &Filesystem{SourceSpec: sp, fi: fi}
+}
+
// Files returns a slice of readable files.
-func (f *Filesystem) Files() []ReadableFile {
+func (f *Filesystem) Files() ([]File, error) {
f.filesInit.Do(func() {
- f.captureFiles()
+ err := f.captureFiles()
+ if err != nil {
+ f.filesInitErr = errors.Wrap(err, "capture files")
+ }
})
- return f.files
+ return f.files, f.filesInitErr
}
// add populates a file in the Filesystem.files
-func (f *Filesystem) add(name string, fi os.FileInfo) (err error) {
- var file ReadableFile
+func (f *Filesystem) add(name string, fi hugofs.FileMetaInfo) (err error) {
+ var file File
if runtime.GOOS == "darwin" {
// When a file system is HFS+, its filepath is in NFD form.
- name = norm.NFC.String(name)
+ // TODO(bep) mod move this to hugofs name = norm.NFC.String(name)
+ }
+
+ file, err = f.SourceSpec.NewFileInfo(fi)
+ if err != nil {
+ return err
}
- file = f.SourceSpec.NewFileInfo(f.Base, name, false, fi)
f.files = append(f.files, file)
return err
}
-func (f *Filesystem) captureFiles() {
- walker := func(filePath string, fi os.FileInfo, err error) error {
+func (f *Filesystem) captureFiles() error {
+ walker := func(path string, fi hugofs.FileMetaInfo, err error) error {
if err != nil {
+ return err
+ }
+
+ if fi.IsDir() {
return nil
}
- b, err := f.shouldRead(filePath, fi)
+ meta := fi.Meta()
+ filename := meta.Filename()
+
+ b, err := f.shouldRead(filename, fi)
if err != nil {
return err
}
+
if b {
- f.add(filePath, fi)
+ err = f.add(filename, fi)
}
+
return err
}
- if f.SourceFs == nil {
- panic("Must have a fs")
- }
- err := helpers.SymbolicWalk(f.SourceFs, f.Base, walker)
+ w := hugofs.NewWalkway(hugofs.WalkwayConfig{
+ Fs: f.SourceFs,
+ Info: f.fi,
+ Root: f.Base,
+ WalkFn: walker,
+ })
- if err != nil {
- jww.ERROR.Println(err)
- }
+ return w.Walk()
}
-func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) {
+func (f *Filesystem) shouldRead(filename string, fi hugofs.FileMetaInfo) (bool, error) {
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
link, err := filepath.EvalSymlinks(filename)
if err != nil {
jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err)
return false, nil
}
+ // TODO(bep) mod this vs root mapping
linkfi, err := f.SourceFs.Stat(link)
if err != nil {
jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
@@ -113,7 +134,7 @@ func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) {
return false, nil
}
- ignore := f.SourceSpec.IgnoreFile(filename)
+ ignore := f.SourceSpec.IgnoreFile(fi.Meta().Filename())
if fi.IsDir() {
if ignore {
diff --git a/source/filesystem_test.go b/source/filesystem_test.go
index 8c8e30413f2..416799c9ddc 100644
--- a/source/filesystem_test.go
+++ b/source/filesystem_test.go
@@ -14,30 +14,44 @@
package source
import (
- "os"
+ "fmt"
+ "path/filepath"
"runtime"
"testing"
+ "github.com/gohugoio/hugo/modules"
+
+ "github.com/gohugoio/hugo/langs"
+
+ "github.com/spf13/afero"
+
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
+ "github.com/stretchr/testify/require"
"github.com/spf13/viper"
)
func TestEmptySourceFilesystem(t *testing.T) {
+ assert := require.New(t)
ss := newTestSourceSpec()
- src := ss.NewFilesystem("Empty")
- if len(src.Files()) != 0 {
+ src := ss.NewFilesystem("")
+ files, err := src.Files()
+ assert.NoError(err)
+ if len(files) != 0 {
t.Errorf("new filesystem should contain 0 files.")
}
}
-func TestUnicodeNorm(t *testing.T) {
+// TODO(bep) mod fix me; think a little where to apply this.
+func _TestUnicodeNorm(t *testing.T) {
if runtime.GOOS != "darwin" {
// Normalization code is only for Mac OS, since it is not necessary for other OSes.
return
}
+ assert := require.New(t)
+
paths := []struct {
NFC string
NFD string
@@ -47,14 +61,18 @@ func TestUnicodeNorm(t *testing.T) {
}
ss := newTestSourceSpec()
- var fi os.FileInfo
+ fi := hugofs.NewFileMetaInfo(nil, hugofs.FileMeta{})
- for _, path := range paths {
- src := ss.NewFilesystem("base")
+ for i, path := range paths {
+ base := fmt.Sprintf("base%d", i)
+ assert.NoError(afero.WriteFile(ss.Fs.Source, filepath.Join(base, path.NFD), []byte("some data"), 0777))
+ src := ss.NewFilesystem(base)
_ = src.add(path.NFD, fi)
- f := src.Files()[0]
+ files, err := src.Files()
+ assert.NoError(err)
+ f := files[0]
if f.BaseFileName() != path.NFC {
- t.Fatalf("file name in NFD form should be normalized (%s)", path.NFC)
+ t.Fatalf("file %q name in NFD form should be normalized (%s)", f.BaseFileName(), path.NFC)
}
}
@@ -70,12 +88,22 @@ func newTestConfig() *viper.Viper {
v.Set("resourceDir", "resources")
v.Set("publishDir", "public")
v.Set("assetDir", "assets")
+ _, err := langs.LoadLanguageSettings(v, nil)
+ if err != nil {
+ panic(err)
+ }
+ mod, err := modules.CreateProjectModule(v)
+ if err != nil {
+ panic(err)
+ }
+ v.Set("allModules", modules.Modules{mod})
+
return v
}
func newTestSourceSpec() *SourceSpec {
v := newTestConfig()
- fs := hugofs.NewMem(v)
+ fs := hugofs.NewFrom(hugofs.NewBaseFileDecorator(afero.NewMemMapFs()), v)
ps, err := helpers.NewPathSpec(fs, v)
if err != nil {
panic(err)
diff --git a/tpl/cast/docshelper.go b/tpl/cast/docshelper.go
index 6fc35f3c72d..1ee614b10c5 100644
--- a/tpl/cast/docshelper.go
+++ b/tpl/cast/docshelper.go
@@ -17,7 +17,7 @@ import (
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/docshelper"
- "github.com/gohugoio/hugo/htesting"
+ "github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/tpl/internal"
"github.com/spf13/viper"
)
@@ -30,7 +30,7 @@ func init() {
Cfg: viper.New(),
Log: loggers.NewErrorLogger(),
BuildStartListeners: &deps.Listeners{},
- Site: htesting.NewTestHugoSite(),
+ Site: page.NewDummyHugoSite(newTestConfig()),
}
var namespaces internal.TemplateFuncsNamespaces
@@ -47,3 +47,9 @@ func init() {
docshelper.AddDocProvider("tpl", docsProvider)
}
+
+func newTestConfig() *viper.Viper {
+ v := viper.New()
+ v.Set("contentDir", "content")
+ return v
+}
diff --git a/tpl/data/init_test.go b/tpl/data/init_test.go
index c4751e8925e..94c8408ea6c 100644
--- a/tpl/data/init_test.go
+++ b/tpl/data/init_test.go
@@ -16,6 +16,7 @@ package data
import (
"testing"
+ "github.com/gohugoio/hugo/langs"
"github.com/gohugoio/hugo/tpl/internal"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
@@ -27,6 +28,7 @@ func TestInit(t *testing.T) {
v := viper.New()
v.Set("contentDir", "content")
+ langs.LoadLanguageSettings(v, nil)
for _, nsf := range internal.TemplateFuncsNamespaceRegistry {
ns = nsf(newDeps(v))
diff --git a/tpl/data/resources_test.go b/tpl/data/resources_test.go
index a42232f940b..dc436dce6ae 100644
--- a/tpl/data/resources_test.go
+++ b/tpl/data/resources_test.go
@@ -23,6 +23,8 @@ import (
"testing"
"time"
+ "github.com/gohugoio/hugo/modules"
+
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/cache/filecache"
@@ -186,14 +188,19 @@ func newDeps(cfg config.Provider) *deps.Deps {
cfg.Set("layoutDir", "layouts")
cfg.Set("archetypeDir", "archetypes")
- l := langs.NewLanguage("en", cfg)
- l.Set("i18nDir", "i18n")
- cs, err := helpers.NewContentSpec(l)
+ langs.LoadLanguageSettings(cfg, nil)
+ mod, err := modules.CreateProjectModule(cfg)
+ if err != nil {
+ panic(err)
+ }
+ cfg.Set("allModules", modules.Modules{mod})
+
+ cs, err := helpers.NewContentSpec(cfg)
if err != nil {
panic(err)
}
- fs := hugofs.NewMem(l)
+ fs := hugofs.NewMem(cfg)
logger := loggers.NewErrorLogger()
p, err := helpers.NewPathSpec(fs, cfg)
diff --git a/tpl/hugo/init_test.go b/tpl/hugo/init_test.go
index 128f6fc19f0..f4e31f622cb 100644
--- a/tpl/hugo/init_test.go
+++ b/tpl/hugo/init_test.go
@@ -16,17 +16,19 @@ package hugo
import (
"testing"
- "github.com/gohugoio/hugo/htesting"
-
"github.com/gohugoio/hugo/deps"
+ "github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/tpl/internal"
+ "github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
func TestInit(t *testing.T) {
var found bool
var ns *internal.TemplateFuncsNamespace
- s := htesting.NewTestHugoSite()
+ v := viper.New()
+ v.Set("contentDir", "content")
+ s := page.NewDummyHugoSite(v)
for _, nsf := range internal.TemplateFuncsNamespaceRegistry {
ns = nsf(&deps.Deps{Site: s})
diff --git a/tpl/os/os.go b/tpl/os/os.go
index 2dab5c4906a..28731e53318 100644
--- a/tpl/os/os.go
+++ b/tpl/os/os.go
@@ -18,6 +18,7 @@ package os
import (
"errors"
"fmt"
+ "os"
_os "os"
"github.com/gohugoio/hugo/deps"
@@ -26,23 +27,21 @@ import (
)
// New returns a new instance of the os-namespaced template functions.
-func New(deps *deps.Deps) *Namespace {
+func New(d *deps.Deps) *Namespace {
- // Since Hugo 0.38 we can have multiple content dirs. This can make it hard to
- // reason about where the file is placed relative to the project root.
- // To make the {{ readFile .Filename }} variant just work, we create a composite
- // filesystem that first checks the work dir fs and then the content fs.
var rfs afero.Fs
- if deps.Fs != nil {
- rfs = deps.Fs.WorkingDir
- if deps.PathSpec != nil && deps.PathSpec.BaseFs != nil {
- rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(deps.PathSpec.BaseFs.Content.Fs, deps.Fs.WorkingDir))
+ if d.Fs != nil {
+ // TODO(bep) mod consider afero.NewCopyOnWriteFs(BaseFs.Work.Fs. ...)
+ rfs = d.Fs.WorkingDir
+ if d.PathSpec != nil && d.PathSpec.BaseFs != nil {
+ rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(d.PathSpec.BaseFs.Content.Fs, d.Fs.WorkingDir))
}
+
}
return &Namespace{
readFileFs: rfs,
- deps: deps,
+ deps: d,
}
}
@@ -76,6 +75,9 @@ func readFile(fs afero.Fs, filename string) (string, error) {
return "", fmt.Errorf("file %q is too big", filename)
}
} else {
+ if os.IsNotExist(err) {
+ return "", fmt.Errorf("file %q does not exist", filename)
+ }
return "", err
}
b, err := afero.ReadFile(fs, filename)
@@ -96,6 +98,10 @@ func (ns *Namespace) ReadFile(i interface{}) (string, error) {
return "", err
}
+ if ns.deps.PathSpec != nil {
+ s = ns.deps.PathSpec.RelPathify(s)
+ }
+
return readFile(ns.readFileFs, s)
}
diff --git a/tpl/site/init_test.go b/tpl/site/init_test.go
index 00704d94355..5ef8856770e 100644
--- a/tpl/site/init_test.go
+++ b/tpl/site/init_test.go
@@ -16,8 +16,10 @@ package site
import (
"testing"
+ "github.com/spf13/viper"
+
"github.com/gohugoio/hugo/deps"
- "github.com/gohugoio/hugo/htesting"
+ "github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/tpl/internal"
"github.com/stretchr/testify/require"
)
@@ -25,7 +27,9 @@ import (
func TestInit(t *testing.T) {
var found bool
var ns *internal.TemplateFuncsNamespace
- s := htesting.NewTestHugoSite()
+ v := viper.New()
+ v.Set("contentDir", "content")
+ s := page.NewDummyHugoSite(v)
for _, nsf := range internal.TemplateFuncsNamespaceRegistry {
ns = nsf(&deps.Deps{Site: s})
diff --git a/tpl/template.go b/tpl/template.go
index 93577136407..cd00d8061bc 100644
--- a/tpl/template.go
+++ b/tpl/template.go
@@ -252,12 +252,15 @@ func (t *TemplateAdapter) fileAndFilename(name string) (afero.File, string, erro
if err != nil {
return nil, "", err
}
- f, err := fs.Open(filename)
+ fim := fi.(hugofs.FileMetaInfo)
+ meta := fim.Meta()
+
+ f, err := meta.Open()
if err != nil {
return nil, "", errors.Wrapf(err, "failed to open template file %q:", filename)
}
- return f, fi.(hugofs.RealFilenameInfo).RealFilename(), nil
+ return f, meta.Filename(), nil
}
// ExecuteToString executes the current template and returns the result as a
diff --git a/tpl/tplimpl/embedded/templates.autogen.go b/tpl/tplimpl/embedded/templates.autogen.go
index cee4276c96d..d0428f910a3 100644
--- a/tpl/tplimpl/embedded/templates.autogen.go
+++ b/tpl/tplimpl/embedded/templates.autogen.go
@@ -269,8 +269,7 @@ if (!doNotTrack) {
{{ end }}
{{ end }}`},
- {`schema.html`, `
-
+ {`schema.html`, `
{{if .IsPage}}{{ $ISO8601 := "2006-01-02T15:04:05-07:00" }}{{ if not .PublishDate.IsZero }}
diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go
index f0d3066e2ec..cce867ac2ba 100644
--- a/tpl/tplimpl/template.go
+++ b/tpl/tplimpl/template.go
@@ -711,7 +711,7 @@ func (t *templateHandler) RebuildClone() {
func (t *templateHandler) loadTemplates(prefix string) error {
- walker := func(path string, fi os.FileInfo, err error) error {
+ walker := func(path string, fi hugofs.FileMetaInfo, err error) error {
if err != nil || fi.IsDir() {
return err
}
@@ -928,8 +928,8 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e
realFilename := filename
if fi, err := fs.Stat(filename); err == nil {
- if fir, ok := fi.(hugofs.RealFilenameInfo); ok {
- realFilename = fir.RealFilename()
+ if fim, ok := fi.(hugofs.FileMetaInfo); ok {
+ realFilename = fim.Meta().Filename()
}
}
diff --git a/tpl/tplimpl/template_funcs_test.go b/tpl/tplimpl/template_funcs_test.go
index 449d20fd4c5..faf5b01fe45 100644
--- a/tpl/tplimpl/template_funcs_test.go
+++ b/tpl/tplimpl/template_funcs_test.go
@@ -21,7 +21,9 @@ import (
"testing"
"time"
- "github.com/gohugoio/hugo/htesting"
+ "github.com/gohugoio/hugo/modules"
+
+ "github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/common/loggers"
@@ -52,6 +54,14 @@ func newTestConfig() config.Provider {
v.Set("assetDir", "assets")
v.Set("resourceDir", "resources")
v.Set("publishDir", "public")
+
+ langs.LoadLanguageSettings(v, nil)
+ mod, err := modules.CreateProjectModule(v)
+ if err != nil {
+ panic(err)
+ }
+ v.Set("allModules", modules.Modules{mod})
+
return v
}
@@ -59,7 +69,7 @@ func newDepsConfig(cfg config.Provider) deps.DepsCfg {
l := langs.NewLanguage("en", cfg)
return deps.DepsCfg{
Language: l,
- Site: htesting.NewTestHugoSite(),
+ Site: page.NewDummyHugoSite(cfg),
Cfg: cfg,
Fs: hugofs.NewMem(l),
Logger: logger,