Optimize the multilanguage build process

Work In Progress!

This commit makes a rework of the build and rebuild process to better suit a multi-site setup.

This also includes a complete overhaul of the site tests. Previous these were a messy mix that
were testing just small parts of the build chain, some of it testing code-paths not even used in
"real life". Now all tests that depends on a built site follows the same and real production code path.

See #2309

commands/hugo.go

@@ -49,7 +49,7 @@ import (
 // Hugo represents the Hugo sites to build. This variable is exported as it
 // is used by at least one external library (the Hugo caddy plugin). We should
 // provide a cleaner external API, but until then, this is it.
-var Hugo hugolib.HugoSites
+var Hugo *hugolib.HugoSites
 
 // userError is an error used to signal different error situations in command handling.
 type commandError struct {
@@ -701,11 +701,11 @@ func getDirList() []string {
 func buildSites(watching ...bool) (err error) {
 	fmt.Println("Started building sites ...")
 	w := len(watching) > 0 && watching[0]
-	return Hugo.Build(w, true)
+	return Hugo.Build(hugolib.BuildCfg{Watching: w, PrintStats: true})
 }
 
 func rebuildSites(events []fsnotify.Event) error {
-	return Hugo.Rebuild(events, true)
+	return Hugo.Rebuild(hugolib.BuildCfg{PrintStats: true}, events...)
 }
 
 // NewWatcher creates a new watcher to watch filesystem events.

commands/list.go

@@ -53,7 +53,7 @@ var listDraftsCmd = &cobra.Command{
 
 		site := &hugolib.Site{}
 
-		if err := site.Process(); err != nil {
+		if err := site.PreProcess(hugolib.BuildCfg{}); err != nil {
 			return newSystemError("Error Processing Source Content", err)
 		}
 
@@ -84,7 +84,7 @@ posted in the future.`,
 
 		site := &hugolib.Site{}
 
-		if err := site.Process(); err != nil {
+		if err := site.PreProcess(hugolib.BuildCfg{}); err != nil {
 			return newSystemError("Error Processing Source Content", err)
 		}
 
@@ -115,7 +115,7 @@ expired.`,
 
 		site := &hugolib.Site{}
 
-		if err := site.Process(); err != nil {
+		if err := site.PreProcess(hugolib.BuildCfg{}); err != nil {
 			return newSystemError("Error Processing Source Content", err)
 		}
 

commands/multilingual.go

@@ -11,30 +11,31 @@ import (
 	"github.com/spf13/viper"
 )
 
-func readMultilingualConfiguration() (hugolib.HugoSites, error) {
-	h := make(hugolib.HugoSites, 0)
+func readMultilingualConfiguration() (*hugolib.HugoSites, error) {
+	sites := make([]*hugolib.Site, 0)
 	multilingual := viper.GetStringMap("Multilingual")
 	if len(multilingual) == 0 {
 		// TODO(bep) multilingo langConfigsList = append(langConfigsList, hugolib.NewLanguage("en"))
-		h = append(h, hugolib.NewSite(hugolib.NewLanguage("en")))
-		return h, nil
+		sites = append(sites, hugolib.NewSite(hugolib.NewLanguage("en")))
 	}
 
-	var err error
+	if len(multilingual) > 0 {
+		var err error
 
-	langConfigsList, err := toSortedLanguages(multilingual)
+		languages, err := toSortedLanguages(multilingual)
 
-	if err != nil {
-		return nil, fmt.Errorf("Failed to parse multilingual config: %s", err)
-	}
+		if err != nil {
+			return nil, fmt.Errorf("Failed to parse multilingual config: %s", err)
+		}
+
+		for _, lang := range languages {
+			sites = append(sites, hugolib.NewSite(lang))
+		}
 
-	for _, lang := range langConfigsList {
-		s := hugolib.NewSite(lang)
-		s.SetMultilingualConfig(lang, langConfigsList)
-		h = append(h, s)
 	}
 
-	return h, nil
+	return hugolib.NewHugoSites(sites...)
+
 }
 
 func toSortedLanguages(l map[string]interface{}) (hugolib.Languages, error) {

helpers/url.go

@@ -169,6 +169,17 @@ func AbsURL(path string) string {
 	return MakePermalink(baseURL, path).String()
 }
 
+// IsAbsURL determines whether the given path points to an absolute URL.
+// TODO(bep) ml tests
+func IsAbsURL(path string) bool {
+	url, err := url.Parse(path)
+	if err != nil {
+		return false
+	}
+
+	return url.IsAbs() || strings.HasPrefix(path, "//")
+}
+
 // RelURL creates a URL relative to the BaseURL root.
 // Note: The result URL will not include the context root if canonifyURLs is enabled.
 func RelURL(path string) string {

hugolib/embedded_shortcodes_test.go

@@ -56,8 +56,8 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) {
 	templ := tpl.New()
 	p, _ := pageFromString(simplePageWithURL, path)
 	p.Node.Site = &SiteInfo{
-		AllPages: &(Pages{p}),
-		BaseURL:  template.URL(helpers.SanitizeURLKeepTrailingSlash(baseURL)),
+		rawAllPages: &(Pages{p}),
+		BaseURL:     template.URL(helpers.SanitizeURLKeepTrailingSlash(baseURL)),
 	}
 
 	output, err := HandleShortcodes(in, p, templ)
@@ -72,8 +72,7 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) {
 }
 
 func TestShortcodeHighlight(t *testing.T) {
-	viper.Reset()
-	defer viper.Reset()
+	testCommonResetState()
 
 	if !helpers.HasPygments() {
 		t.Skip("Skip test as Pygments is not installed")

hugolib/handler_test.go

@@ -25,8 +25,7 @@ import (
 )
 
 func TestDefaultHandler(t *testing.T) {
-	viper.Reset()
-	defer viper.Reset()
+	testCommonResetState()
 
 	hugofs.InitMemFs()
 	sources := []source.ByteSource{
@@ -45,33 +44,30 @@ func TestDefaultHandler(t *testing.T) {
 	viper.Set("verbose", true)
 
 	s := &Site{
-		Source:  &source.InMemorySource{ByteSource: sources},
-		targets: targetList{page: &target.PagePub{UglyURLs: true}},
-		Lang:    NewLanguage("en"),
+		Source:   &source.InMemorySource{ByteSource: sources},
+		targets:  targetList{page: &target.PagePub{UglyURLs: true, PublishDir: "public"}},
+		Language: NewLanguage("en"),
 	}
 
-	s.initializeSiteInfo()
-
-	s.prepTemplates(
+	if err := buildAndRenderSite(s,
 		"_default/single.html", "{{.Content}}",
 		"head", "<head><script src=\"script.js\"></script></head>",
-		"head_abs", "<head><script src=\"/script.js\"></script></head>")
-
-	// From site_test.go
-	createAndRenderPages(t, s)
+		"head_abs", "<head><script src=\"/script.js\"></script></head>"); err != nil {
+		t.Fatalf("Failed to render site: %s", err)
+	}
 
 	tests := []struct {
 		doc      string
 		expected string
 	}{
-		{filepath.FromSlash("sect/doc1.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"},
-		{filepath.FromSlash("sect/doc2.html"), "<!doctype html><html><body>more content</body></html>"},
-		{filepath.FromSlash("sect/doc3.html"), "\n\n<h1 id=\"doc3\">doc3</h1>\n\n<p><em>some</em> content</p>\n"},
-		{filepath.FromSlash("sect/doc3/img1.png"), string([]byte("‰PNG  ��� IHDR����������:~›U��� IDATWcø��ZMoñ����IEND®B`‚"))},
-		{filepath.FromSlash("sect/img2.gif"), string([]byte("GIF89a��€��ÿÿÿ���,�������D�;"))},
-		{filepath.FromSlash("sect/img2.spf"), string([]byte("****FAKE-FILETYPE****"))},
-		{filepath.FromSlash("doc7.html"), "<html><body>doc7 content</body></html>"},
-		{filepath.FromSlash("sect/doc8.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"},
+		{filepath.FromSlash("public/sect/doc1.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"},
+		{filepath.FromSlash("public/sect/doc2.html"), "<!doctype html><html><body>more content</body></html>"},
+		{filepath.FromSlash("public/sect/doc3.html"), "\n\n<h1 id=\"doc3\">doc3</h1>\n\n<p><em>some</em> content</p>\n"},
+		{filepath.FromSlash("public/sect/doc3/img1.png"), string([]byte("‰PNG  ��� IHDR����������:~›U��� IDATWcø��ZMoñ����IEND®B`‚"))},
+		{filepath.FromSlash("public/sect/img2.gif"), string([]byte("GIF89a��€��ÿÿÿ���,�������D�;"))},
+		{filepath.FromSlash("public/sect/img2.spf"), string([]byte("****FAKE-FILETYPE****"))},
+		{filepath.FromSlash("public/doc7.html"), "<html><body>doc7 content</body></html>"},
+		{filepath.FromSlash("public/sect/doc8.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"},
 	}
 
 	for _, test := range tests {